1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/module.h> 34 #include <rdma/ib_umem.h> 35 #include <rdma/ib_cache.h> 36 #include <rdma/ib_user_verbs.h> 37 #include "mlx5_ib.h" 38 39 /* not supported currently */ 40 static int wq_signature; 41 42 enum { 43 MLX5_IB_ACK_REQ_FREQ = 8, 44 }; 45 46 enum { 47 MLX5_IB_DEFAULT_SCHED_QUEUE = 0x83, 48 MLX5_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f, 49 MLX5_IB_LINK_TYPE_IB = 0, 50 MLX5_IB_LINK_TYPE_ETH = 1 51 }; 52 53 enum { 54 MLX5_IB_SQ_STRIDE = 6, 55 MLX5_IB_CACHE_LINE_SIZE = 64, 56 }; 57 58 static const u32 mlx5_ib_opcode[] = { 59 [IB_WR_SEND] = MLX5_OPCODE_SEND, 60 [IB_WR_LSO] = MLX5_OPCODE_LSO, 61 [IB_WR_SEND_WITH_IMM] = MLX5_OPCODE_SEND_IMM, 62 [IB_WR_RDMA_WRITE] = MLX5_OPCODE_RDMA_WRITE, 63 [IB_WR_RDMA_WRITE_WITH_IMM] = MLX5_OPCODE_RDMA_WRITE_IMM, 64 [IB_WR_RDMA_READ] = MLX5_OPCODE_RDMA_READ, 65 [IB_WR_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_CS, 66 [IB_WR_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_FA, 67 [IB_WR_SEND_WITH_INV] = MLX5_OPCODE_SEND_INVAL, 68 [IB_WR_LOCAL_INV] = MLX5_OPCODE_UMR, 69 [IB_WR_REG_MR] = MLX5_OPCODE_UMR, 70 [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_MASKED_CS, 71 [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_MASKED_FA, 72 [MLX5_IB_WR_UMR] = MLX5_OPCODE_UMR, 73 }; 74 75 struct mlx5_wqe_eth_pad { 76 u8 rsvd0[16]; 77 }; 78 79 enum raw_qp_set_mask_map { 80 MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID = 1UL << 0, 81 }; 82 83 struct mlx5_modify_raw_qp_param { 84 u16 operation; 85 86 u32 set_mask; /* raw_qp_set_mask_map */ 87 u8 rq_q_ctr_id; 88 }; 89 90 static void get_cqs(enum ib_qp_type qp_type, 91 struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq, 92 struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq); 93 94 static int is_qp0(enum ib_qp_type qp_type) 95 { 96 return qp_type == IB_QPT_SMI; 97 } 98 99 static int is_sqp(enum ib_qp_type qp_type) 100 { 101 return is_qp0(qp_type) || is_qp1(qp_type); 102 } 103 104 static void *get_wqe(struct mlx5_ib_qp *qp, int offset) 105 { 106 return mlx5_buf_offset(&qp->buf, offset); 107 } 108 109 static void *get_recv_wqe(struct mlx5_ib_qp *qp, int n) 110 { 111 return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift)); 112 } 113 114 void *mlx5_get_send_wqe(struct mlx5_ib_qp *qp, int n) 115 { 116 return get_wqe(qp, qp->sq.offset + (n << MLX5_IB_SQ_STRIDE)); 117 } 118 119 /** 120 * mlx5_ib_read_user_wqe() - Copy a user-space WQE to kernel space. 121 * 122 * @qp: QP to copy from. 123 * @send: copy from the send queue when non-zero, use the receive queue 124 * otherwise. 125 * @wqe_index: index to start copying from. For send work queues, the 126 * wqe_index is in units of MLX5_SEND_WQE_BB. 127 * For receive work queue, it is the number of work queue 128 * element in the queue. 129 * @buffer: destination buffer. 130 * @length: maximum number of bytes to copy. 131 * 132 * Copies at least a single WQE, but may copy more data. 133 * 134 * Return: the number of bytes copied, or an error code. 135 */ 136 int mlx5_ib_read_user_wqe(struct mlx5_ib_qp *qp, int send, int wqe_index, 137 void *buffer, u32 length, 138 struct mlx5_ib_qp_base *base) 139 { 140 struct ib_device *ibdev = qp->ibqp.device; 141 struct mlx5_ib_dev *dev = to_mdev(ibdev); 142 struct mlx5_ib_wq *wq = send ? &qp->sq : &qp->rq; 143 size_t offset; 144 size_t wq_end; 145 struct ib_umem *umem = base->ubuffer.umem; 146 u32 first_copy_length; 147 int wqe_length; 148 int ret; 149 150 if (wq->wqe_cnt == 0) { 151 mlx5_ib_dbg(dev, "mlx5_ib_read_user_wqe for a QP with wqe_cnt == 0. qp_type: 0x%x\n", 152 qp->ibqp.qp_type); 153 return -EINVAL; 154 } 155 156 offset = wq->offset + ((wqe_index % wq->wqe_cnt) << wq->wqe_shift); 157 wq_end = wq->offset + (wq->wqe_cnt << wq->wqe_shift); 158 159 if (send && length < sizeof(struct mlx5_wqe_ctrl_seg)) 160 return -EINVAL; 161 162 if (offset > umem->length || 163 (send && offset + sizeof(struct mlx5_wqe_ctrl_seg) > umem->length)) 164 return -EINVAL; 165 166 first_copy_length = min_t(u32, offset + length, wq_end) - offset; 167 ret = ib_umem_copy_from(buffer, umem, offset, first_copy_length); 168 if (ret) 169 return ret; 170 171 if (send) { 172 struct mlx5_wqe_ctrl_seg *ctrl = buffer; 173 int ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK; 174 175 wqe_length = ds * MLX5_WQE_DS_UNITS; 176 } else { 177 wqe_length = 1 << wq->wqe_shift; 178 } 179 180 if (wqe_length <= first_copy_length) 181 return first_copy_length; 182 183 ret = ib_umem_copy_from(buffer + first_copy_length, umem, wq->offset, 184 wqe_length - first_copy_length); 185 if (ret) 186 return ret; 187 188 return wqe_length; 189 } 190 191 static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type) 192 { 193 struct ib_qp *ibqp = &to_mibqp(qp)->ibqp; 194 struct ib_event event; 195 196 if (type == MLX5_EVENT_TYPE_PATH_MIG) { 197 /* This event is only valid for trans_qps */ 198 to_mibqp(qp)->port = to_mibqp(qp)->trans_qp.alt_port; 199 } 200 201 if (ibqp->event_handler) { 202 event.device = ibqp->device; 203 event.element.qp = ibqp; 204 switch (type) { 205 case MLX5_EVENT_TYPE_PATH_MIG: 206 event.event = IB_EVENT_PATH_MIG; 207 break; 208 case MLX5_EVENT_TYPE_COMM_EST: 209 event.event = IB_EVENT_COMM_EST; 210 break; 211 case MLX5_EVENT_TYPE_SQ_DRAINED: 212 event.event = IB_EVENT_SQ_DRAINED; 213 break; 214 case MLX5_EVENT_TYPE_SRQ_LAST_WQE: 215 event.event = IB_EVENT_QP_LAST_WQE_REACHED; 216 break; 217 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR: 218 event.event = IB_EVENT_QP_FATAL; 219 break; 220 case MLX5_EVENT_TYPE_PATH_MIG_FAILED: 221 event.event = IB_EVENT_PATH_MIG_ERR; 222 break; 223 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR: 224 event.event = IB_EVENT_QP_REQ_ERR; 225 break; 226 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR: 227 event.event = IB_EVENT_QP_ACCESS_ERR; 228 break; 229 default: 230 pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n", type, qp->qpn); 231 return; 232 } 233 234 ibqp->event_handler(&event, ibqp->qp_context); 235 } 236 } 237 238 static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap, 239 int has_rq, struct mlx5_ib_qp *qp, struct mlx5_ib_create_qp *ucmd) 240 { 241 int wqe_size; 242 int wq_size; 243 244 /* Sanity check RQ size before proceeding */ 245 if (cap->max_recv_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) 246 return -EINVAL; 247 248 if (!has_rq) { 249 qp->rq.max_gs = 0; 250 qp->rq.wqe_cnt = 0; 251 qp->rq.wqe_shift = 0; 252 cap->max_recv_wr = 0; 253 cap->max_recv_sge = 0; 254 } else { 255 if (ucmd) { 256 qp->rq.wqe_cnt = ucmd->rq_wqe_count; 257 qp->rq.wqe_shift = ucmd->rq_wqe_shift; 258 qp->rq.max_gs = (1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) - qp->wq_sig; 259 qp->rq.max_post = qp->rq.wqe_cnt; 260 } else { 261 wqe_size = qp->wq_sig ? sizeof(struct mlx5_wqe_signature_seg) : 0; 262 wqe_size += cap->max_recv_sge * sizeof(struct mlx5_wqe_data_seg); 263 wqe_size = roundup_pow_of_two(wqe_size); 264 wq_size = roundup_pow_of_two(cap->max_recv_wr) * wqe_size; 265 wq_size = max_t(int, wq_size, MLX5_SEND_WQE_BB); 266 qp->rq.wqe_cnt = wq_size / wqe_size; 267 if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq)) { 268 mlx5_ib_dbg(dev, "wqe_size %d, max %d\n", 269 wqe_size, 270 MLX5_CAP_GEN(dev->mdev, 271 max_wqe_sz_rq)); 272 return -EINVAL; 273 } 274 qp->rq.wqe_shift = ilog2(wqe_size); 275 qp->rq.max_gs = (1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) - qp->wq_sig; 276 qp->rq.max_post = qp->rq.wqe_cnt; 277 } 278 } 279 280 return 0; 281 } 282 283 static int sq_overhead(struct ib_qp_init_attr *attr) 284 { 285 int size = 0; 286 287 switch (attr->qp_type) { 288 case IB_QPT_XRC_INI: 289 size += sizeof(struct mlx5_wqe_xrc_seg); 290 /* fall through */ 291 case IB_QPT_RC: 292 size += sizeof(struct mlx5_wqe_ctrl_seg) + 293 max(sizeof(struct mlx5_wqe_atomic_seg) + 294 sizeof(struct mlx5_wqe_raddr_seg), 295 sizeof(struct mlx5_wqe_umr_ctrl_seg) + 296 sizeof(struct mlx5_mkey_seg)); 297 break; 298 299 case IB_QPT_XRC_TGT: 300 return 0; 301 302 case IB_QPT_UC: 303 size += sizeof(struct mlx5_wqe_ctrl_seg) + 304 max(sizeof(struct mlx5_wqe_raddr_seg), 305 sizeof(struct mlx5_wqe_umr_ctrl_seg) + 306 sizeof(struct mlx5_mkey_seg)); 307 break; 308 309 case IB_QPT_UD: 310 if (attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO) 311 size += sizeof(struct mlx5_wqe_eth_pad) + 312 sizeof(struct mlx5_wqe_eth_seg); 313 /* fall through */ 314 case IB_QPT_SMI: 315 case MLX5_IB_QPT_HW_GSI: 316 size += sizeof(struct mlx5_wqe_ctrl_seg) + 317 sizeof(struct mlx5_wqe_datagram_seg); 318 break; 319 320 case MLX5_IB_QPT_REG_UMR: 321 size += sizeof(struct mlx5_wqe_ctrl_seg) + 322 sizeof(struct mlx5_wqe_umr_ctrl_seg) + 323 sizeof(struct mlx5_mkey_seg); 324 break; 325 326 default: 327 return -EINVAL; 328 } 329 330 return size; 331 } 332 333 static int calc_send_wqe(struct ib_qp_init_attr *attr) 334 { 335 int inl_size = 0; 336 int size; 337 338 size = sq_overhead(attr); 339 if (size < 0) 340 return size; 341 342 if (attr->cap.max_inline_data) { 343 inl_size = size + sizeof(struct mlx5_wqe_inline_seg) + 344 attr->cap.max_inline_data; 345 } 346 347 size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg); 348 if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN && 349 ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB) < MLX5_SIG_WQE_SIZE) 350 return MLX5_SIG_WQE_SIZE; 351 else 352 return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB); 353 } 354 355 static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr, 356 struct mlx5_ib_qp *qp) 357 { 358 int wqe_size; 359 int wq_size; 360 361 if (!attr->cap.max_send_wr) 362 return 0; 363 364 wqe_size = calc_send_wqe(attr); 365 mlx5_ib_dbg(dev, "wqe_size %d\n", wqe_size); 366 if (wqe_size < 0) 367 return wqe_size; 368 369 if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) { 370 mlx5_ib_dbg(dev, "wqe_size(%d) > max_sq_desc_sz(%d)\n", 371 wqe_size, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)); 372 return -EINVAL; 373 } 374 375 qp->max_inline_data = wqe_size - sq_overhead(attr) - 376 sizeof(struct mlx5_wqe_inline_seg); 377 attr->cap.max_inline_data = qp->max_inline_data; 378 379 if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN) 380 qp->signature_en = true; 381 382 wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size); 383 qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB; 384 if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) { 385 mlx5_ib_dbg(dev, "wqe count(%d) exceeds limits(%d)\n", 386 qp->sq.wqe_cnt, 387 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz)); 388 return -ENOMEM; 389 } 390 qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB); 391 qp->sq.max_gs = attr->cap.max_send_sge; 392 qp->sq.max_post = wq_size / wqe_size; 393 attr->cap.max_send_wr = qp->sq.max_post; 394 395 return wq_size; 396 } 397 398 static int set_user_buf_size(struct mlx5_ib_dev *dev, 399 struct mlx5_ib_qp *qp, 400 struct mlx5_ib_create_qp *ucmd, 401 struct mlx5_ib_qp_base *base, 402 struct ib_qp_init_attr *attr) 403 { 404 int desc_sz = 1 << qp->sq.wqe_shift; 405 406 if (desc_sz > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) { 407 mlx5_ib_warn(dev, "desc_sz %d, max_sq_desc_sz %d\n", 408 desc_sz, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)); 409 return -EINVAL; 410 } 411 412 if (ucmd->sq_wqe_count && ((1 << ilog2(ucmd->sq_wqe_count)) != ucmd->sq_wqe_count)) { 413 mlx5_ib_warn(dev, "sq_wqe_count %d, sq_wqe_count %d\n", 414 ucmd->sq_wqe_count, ucmd->sq_wqe_count); 415 return -EINVAL; 416 } 417 418 qp->sq.wqe_cnt = ucmd->sq_wqe_count; 419 420 if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) { 421 mlx5_ib_warn(dev, "wqe_cnt %d, max_wqes %d\n", 422 qp->sq.wqe_cnt, 423 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz)); 424 return -EINVAL; 425 } 426 427 if (attr->qp_type == IB_QPT_RAW_PACKET) { 428 base->ubuffer.buf_size = qp->rq.wqe_cnt << qp->rq.wqe_shift; 429 qp->raw_packet_qp.sq.ubuffer.buf_size = qp->sq.wqe_cnt << 6; 430 } else { 431 base->ubuffer.buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + 432 (qp->sq.wqe_cnt << 6); 433 } 434 435 return 0; 436 } 437 438 static int qp_has_rq(struct ib_qp_init_attr *attr) 439 { 440 if (attr->qp_type == IB_QPT_XRC_INI || 441 attr->qp_type == IB_QPT_XRC_TGT || attr->srq || 442 attr->qp_type == MLX5_IB_QPT_REG_UMR || 443 !attr->cap.max_recv_wr) 444 return 0; 445 446 return 1; 447 } 448 449 static int first_med_uuar(void) 450 { 451 return 1; 452 } 453 454 static int next_uuar(int n) 455 { 456 n++; 457 458 while (((n % 4) & 2)) 459 n++; 460 461 return n; 462 } 463 464 static int num_med_uuar(struct mlx5_uuar_info *uuari) 465 { 466 int n; 467 468 n = uuari->num_uars * MLX5_NON_FP_BF_REGS_PER_PAGE - 469 uuari->num_low_latency_uuars - 1; 470 471 return n >= 0 ? n : 0; 472 } 473 474 static int max_uuari(struct mlx5_uuar_info *uuari) 475 { 476 return uuari->num_uars * 4; 477 } 478 479 static int first_hi_uuar(struct mlx5_uuar_info *uuari) 480 { 481 int med; 482 int i; 483 int t; 484 485 med = num_med_uuar(uuari); 486 for (t = 0, i = first_med_uuar();; i = next_uuar(i)) { 487 t++; 488 if (t == med) 489 return next_uuar(i); 490 } 491 492 return 0; 493 } 494 495 static int alloc_high_class_uuar(struct mlx5_uuar_info *uuari) 496 { 497 int i; 498 499 for (i = first_hi_uuar(uuari); i < max_uuari(uuari); i = next_uuar(i)) { 500 if (!test_bit(i, uuari->bitmap)) { 501 set_bit(i, uuari->bitmap); 502 uuari->count[i]++; 503 return i; 504 } 505 } 506 507 return -ENOMEM; 508 } 509 510 static int alloc_med_class_uuar(struct mlx5_uuar_info *uuari) 511 { 512 int minidx = first_med_uuar(); 513 int i; 514 515 for (i = first_med_uuar(); i < first_hi_uuar(uuari); i = next_uuar(i)) { 516 if (uuari->count[i] < uuari->count[minidx]) 517 minidx = i; 518 } 519 520 uuari->count[minidx]++; 521 return minidx; 522 } 523 524 static int alloc_uuar(struct mlx5_uuar_info *uuari, 525 enum mlx5_ib_latency_class lat) 526 { 527 int uuarn = -EINVAL; 528 529 mutex_lock(&uuari->lock); 530 switch (lat) { 531 case MLX5_IB_LATENCY_CLASS_LOW: 532 uuarn = 0; 533 uuari->count[uuarn]++; 534 break; 535 536 case MLX5_IB_LATENCY_CLASS_MEDIUM: 537 if (uuari->ver < 2) 538 uuarn = -ENOMEM; 539 else 540 uuarn = alloc_med_class_uuar(uuari); 541 break; 542 543 case MLX5_IB_LATENCY_CLASS_HIGH: 544 if (uuari->ver < 2) 545 uuarn = -ENOMEM; 546 else 547 uuarn = alloc_high_class_uuar(uuari); 548 break; 549 550 case MLX5_IB_LATENCY_CLASS_FAST_PATH: 551 uuarn = 2; 552 break; 553 } 554 mutex_unlock(&uuari->lock); 555 556 return uuarn; 557 } 558 559 static void free_med_class_uuar(struct mlx5_uuar_info *uuari, int uuarn) 560 { 561 clear_bit(uuarn, uuari->bitmap); 562 --uuari->count[uuarn]; 563 } 564 565 static void free_high_class_uuar(struct mlx5_uuar_info *uuari, int uuarn) 566 { 567 clear_bit(uuarn, uuari->bitmap); 568 --uuari->count[uuarn]; 569 } 570 571 static void free_uuar(struct mlx5_uuar_info *uuari, int uuarn) 572 { 573 int nuuars = uuari->num_uars * MLX5_BF_REGS_PER_PAGE; 574 int high_uuar = nuuars - uuari->num_low_latency_uuars; 575 576 mutex_lock(&uuari->lock); 577 if (uuarn == 0) { 578 --uuari->count[uuarn]; 579 goto out; 580 } 581 582 if (uuarn < high_uuar) { 583 free_med_class_uuar(uuari, uuarn); 584 goto out; 585 } 586 587 free_high_class_uuar(uuari, uuarn); 588 589 out: 590 mutex_unlock(&uuari->lock); 591 } 592 593 static enum mlx5_qp_state to_mlx5_state(enum ib_qp_state state) 594 { 595 switch (state) { 596 case IB_QPS_RESET: return MLX5_QP_STATE_RST; 597 case IB_QPS_INIT: return MLX5_QP_STATE_INIT; 598 case IB_QPS_RTR: return MLX5_QP_STATE_RTR; 599 case IB_QPS_RTS: return MLX5_QP_STATE_RTS; 600 case IB_QPS_SQD: return MLX5_QP_STATE_SQD; 601 case IB_QPS_SQE: return MLX5_QP_STATE_SQER; 602 case IB_QPS_ERR: return MLX5_QP_STATE_ERR; 603 default: return -1; 604 } 605 } 606 607 static int to_mlx5_st(enum ib_qp_type type) 608 { 609 switch (type) { 610 case IB_QPT_RC: return MLX5_QP_ST_RC; 611 case IB_QPT_UC: return MLX5_QP_ST_UC; 612 case IB_QPT_UD: return MLX5_QP_ST_UD; 613 case MLX5_IB_QPT_REG_UMR: return MLX5_QP_ST_REG_UMR; 614 case IB_QPT_XRC_INI: 615 case IB_QPT_XRC_TGT: return MLX5_QP_ST_XRC; 616 case IB_QPT_SMI: return MLX5_QP_ST_QP0; 617 case MLX5_IB_QPT_HW_GSI: return MLX5_QP_ST_QP1; 618 case IB_QPT_RAW_IPV6: return MLX5_QP_ST_RAW_IPV6; 619 case IB_QPT_RAW_PACKET: 620 case IB_QPT_RAW_ETHERTYPE: return MLX5_QP_ST_RAW_ETHERTYPE; 621 case IB_QPT_MAX: 622 default: return -EINVAL; 623 } 624 } 625 626 static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq, 627 struct mlx5_ib_cq *recv_cq); 628 static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq, 629 struct mlx5_ib_cq *recv_cq); 630 631 static int uuarn_to_uar_index(struct mlx5_uuar_info *uuari, int uuarn) 632 { 633 return uuari->uars[uuarn / MLX5_BF_REGS_PER_PAGE].index; 634 } 635 636 static int mlx5_ib_umem_get(struct mlx5_ib_dev *dev, 637 struct ib_pd *pd, 638 unsigned long addr, size_t size, 639 struct ib_umem **umem, 640 int *npages, int *page_shift, int *ncont, 641 u32 *offset) 642 { 643 int err; 644 645 *umem = ib_umem_get(pd->uobject->context, addr, size, 0, 0); 646 if (IS_ERR(*umem)) { 647 mlx5_ib_dbg(dev, "umem_get failed\n"); 648 return PTR_ERR(*umem); 649 } 650 651 mlx5_ib_cont_pages(*umem, addr, npages, page_shift, ncont, NULL); 652 653 err = mlx5_ib_get_buf_offset(addr, *page_shift, offset); 654 if (err) { 655 mlx5_ib_warn(dev, "bad offset\n"); 656 goto err_umem; 657 } 658 659 mlx5_ib_dbg(dev, "addr 0x%lx, size %zu, npages %d, page_shift %d, ncont %d, offset %d\n", 660 addr, size, *npages, *page_shift, *ncont, *offset); 661 662 return 0; 663 664 err_umem: 665 ib_umem_release(*umem); 666 *umem = NULL; 667 668 return err; 669 } 670 671 static void destroy_user_rq(struct ib_pd *pd, struct mlx5_ib_rwq *rwq) 672 { 673 struct mlx5_ib_ucontext *context; 674 675 context = to_mucontext(pd->uobject->context); 676 mlx5_ib_db_unmap_user(context, &rwq->db); 677 if (rwq->umem) 678 ib_umem_release(rwq->umem); 679 } 680 681 static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd, 682 struct mlx5_ib_rwq *rwq, 683 struct mlx5_ib_create_wq *ucmd) 684 { 685 struct mlx5_ib_ucontext *context; 686 int page_shift = 0; 687 int npages; 688 u32 offset = 0; 689 int ncont = 0; 690 int err; 691 692 if (!ucmd->buf_addr) 693 return -EINVAL; 694 695 context = to_mucontext(pd->uobject->context); 696 rwq->umem = ib_umem_get(pd->uobject->context, ucmd->buf_addr, 697 rwq->buf_size, 0, 0); 698 if (IS_ERR(rwq->umem)) { 699 mlx5_ib_dbg(dev, "umem_get failed\n"); 700 err = PTR_ERR(rwq->umem); 701 return err; 702 } 703 704 mlx5_ib_cont_pages(rwq->umem, ucmd->buf_addr, &npages, &page_shift, 705 &ncont, NULL); 706 err = mlx5_ib_get_buf_offset(ucmd->buf_addr, page_shift, 707 &rwq->rq_page_offset); 708 if (err) { 709 mlx5_ib_warn(dev, "bad offset\n"); 710 goto err_umem; 711 } 712 713 rwq->rq_num_pas = ncont; 714 rwq->page_shift = page_shift; 715 rwq->log_page_size = page_shift - MLX5_ADAPTER_PAGE_SHIFT; 716 rwq->wq_sig = !!(ucmd->flags & MLX5_WQ_FLAG_SIGNATURE); 717 718 mlx5_ib_dbg(dev, "addr 0x%llx, size %zd, npages %d, page_shift %d, ncont %d, offset %d\n", 719 (unsigned long long)ucmd->buf_addr, rwq->buf_size, 720 npages, page_shift, ncont, offset); 721 722 err = mlx5_ib_db_map_user(context, ucmd->db_addr, &rwq->db); 723 if (err) { 724 mlx5_ib_dbg(dev, "map failed\n"); 725 goto err_umem; 726 } 727 728 rwq->create_type = MLX5_WQ_USER; 729 return 0; 730 731 err_umem: 732 ib_umem_release(rwq->umem); 733 return err; 734 } 735 736 static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, 737 struct mlx5_ib_qp *qp, struct ib_udata *udata, 738 struct ib_qp_init_attr *attr, 739 u32 **in, 740 struct mlx5_ib_create_qp_resp *resp, int *inlen, 741 struct mlx5_ib_qp_base *base) 742 { 743 struct mlx5_ib_ucontext *context; 744 struct mlx5_ib_create_qp ucmd; 745 struct mlx5_ib_ubuffer *ubuffer = &base->ubuffer; 746 int page_shift = 0; 747 int uar_index; 748 int npages; 749 u32 offset = 0; 750 int uuarn; 751 int ncont = 0; 752 __be64 *pas; 753 void *qpc; 754 int err; 755 756 err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)); 757 if (err) { 758 mlx5_ib_dbg(dev, "copy failed\n"); 759 return err; 760 } 761 762 context = to_mucontext(pd->uobject->context); 763 /* 764 * TBD: should come from the verbs when we have the API 765 */ 766 if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL) 767 /* In CROSS_CHANNEL CQ and QP must use the same UAR */ 768 uuarn = MLX5_CROSS_CHANNEL_UUAR; 769 else { 770 uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_HIGH); 771 if (uuarn < 0) { 772 mlx5_ib_dbg(dev, "failed to allocate low latency UUAR\n"); 773 mlx5_ib_dbg(dev, "reverting to medium latency\n"); 774 uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_MEDIUM); 775 if (uuarn < 0) { 776 mlx5_ib_dbg(dev, "failed to allocate medium latency UUAR\n"); 777 mlx5_ib_dbg(dev, "reverting to high latency\n"); 778 uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_LOW); 779 if (uuarn < 0) { 780 mlx5_ib_warn(dev, "uuar allocation failed\n"); 781 return uuarn; 782 } 783 } 784 } 785 } 786 787 uar_index = uuarn_to_uar_index(&context->uuari, uuarn); 788 mlx5_ib_dbg(dev, "uuarn 0x%x, uar_index 0x%x\n", uuarn, uar_index); 789 790 qp->rq.offset = 0; 791 qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB); 792 qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift; 793 794 err = set_user_buf_size(dev, qp, &ucmd, base, attr); 795 if (err) 796 goto err_uuar; 797 798 if (ucmd.buf_addr && ubuffer->buf_size) { 799 ubuffer->buf_addr = ucmd.buf_addr; 800 err = mlx5_ib_umem_get(dev, pd, ubuffer->buf_addr, 801 ubuffer->buf_size, 802 &ubuffer->umem, &npages, &page_shift, 803 &ncont, &offset); 804 if (err) 805 goto err_uuar; 806 } else { 807 ubuffer->umem = NULL; 808 } 809 810 *inlen = MLX5_ST_SZ_BYTES(create_qp_in) + 811 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * ncont; 812 *in = mlx5_vzalloc(*inlen); 813 if (!*in) { 814 err = -ENOMEM; 815 goto err_umem; 816 } 817 818 pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas); 819 if (ubuffer->umem) 820 mlx5_ib_populate_pas(dev, ubuffer->umem, page_shift, pas, 0); 821 822 qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc); 823 824 MLX5_SET(qpc, qpc, log_page_size, page_shift - MLX5_ADAPTER_PAGE_SHIFT); 825 MLX5_SET(qpc, qpc, page_offset, offset); 826 827 MLX5_SET(qpc, qpc, uar_page, uar_index); 828 resp->uuar_index = uuarn; 829 qp->uuarn = uuarn; 830 831 err = mlx5_ib_db_map_user(context, ucmd.db_addr, &qp->db); 832 if (err) { 833 mlx5_ib_dbg(dev, "map failed\n"); 834 goto err_free; 835 } 836 837 err = ib_copy_to_udata(udata, resp, sizeof(*resp)); 838 if (err) { 839 mlx5_ib_dbg(dev, "copy failed\n"); 840 goto err_unmap; 841 } 842 qp->create_type = MLX5_QP_USER; 843 844 return 0; 845 846 err_unmap: 847 mlx5_ib_db_unmap_user(context, &qp->db); 848 849 err_free: 850 kvfree(*in); 851 852 err_umem: 853 if (ubuffer->umem) 854 ib_umem_release(ubuffer->umem); 855 856 err_uuar: 857 free_uuar(&context->uuari, uuarn); 858 return err; 859 } 860 861 static void destroy_qp_user(struct ib_pd *pd, struct mlx5_ib_qp *qp, 862 struct mlx5_ib_qp_base *base) 863 { 864 struct mlx5_ib_ucontext *context; 865 866 context = to_mucontext(pd->uobject->context); 867 mlx5_ib_db_unmap_user(context, &qp->db); 868 if (base->ubuffer.umem) 869 ib_umem_release(base->ubuffer.umem); 870 free_uuar(&context->uuari, qp->uuarn); 871 } 872 873 static int create_kernel_qp(struct mlx5_ib_dev *dev, 874 struct ib_qp_init_attr *init_attr, 875 struct mlx5_ib_qp *qp, 876 u32 **in, int *inlen, 877 struct mlx5_ib_qp_base *base) 878 { 879 enum mlx5_ib_latency_class lc = MLX5_IB_LATENCY_CLASS_LOW; 880 struct mlx5_uuar_info *uuari; 881 int uar_index; 882 void *qpc; 883 int uuarn; 884 int err; 885 886 uuari = &dev->mdev->priv.uuari; 887 if (init_attr->create_flags & ~(IB_QP_CREATE_SIGNATURE_EN | 888 IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK | 889 IB_QP_CREATE_IPOIB_UD_LSO | 890 mlx5_ib_create_qp_sqpn_qp1())) 891 return -EINVAL; 892 893 if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR) 894 lc = MLX5_IB_LATENCY_CLASS_FAST_PATH; 895 896 uuarn = alloc_uuar(uuari, lc); 897 if (uuarn < 0) { 898 mlx5_ib_dbg(dev, "\n"); 899 return -ENOMEM; 900 } 901 902 qp->bf = &uuari->bfs[uuarn]; 903 uar_index = qp->bf->uar->index; 904 905 err = calc_sq_size(dev, init_attr, qp); 906 if (err < 0) { 907 mlx5_ib_dbg(dev, "err %d\n", err); 908 goto err_uuar; 909 } 910 911 qp->rq.offset = 0; 912 qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift; 913 base->ubuffer.buf_size = err + (qp->rq.wqe_cnt << qp->rq.wqe_shift); 914 915 err = mlx5_buf_alloc(dev->mdev, base->ubuffer.buf_size, &qp->buf); 916 if (err) { 917 mlx5_ib_dbg(dev, "err %d\n", err); 918 goto err_uuar; 919 } 920 921 qp->sq.qend = mlx5_get_send_wqe(qp, qp->sq.wqe_cnt); 922 *inlen = MLX5_ST_SZ_BYTES(create_qp_in) + 923 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * qp->buf.npages; 924 *in = mlx5_vzalloc(*inlen); 925 if (!*in) { 926 err = -ENOMEM; 927 goto err_buf; 928 } 929 930 qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc); 931 MLX5_SET(qpc, qpc, uar_page, uar_index); 932 MLX5_SET(qpc, qpc, log_page_size, qp->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 933 934 /* Set "fast registration enabled" for all kernel QPs */ 935 MLX5_SET(qpc, qpc, fre, 1); 936 MLX5_SET(qpc, qpc, rlky, 1); 937 938 if (init_attr->create_flags & mlx5_ib_create_qp_sqpn_qp1()) { 939 MLX5_SET(qpc, qpc, deth_sqpn, 1); 940 qp->flags |= MLX5_IB_QP_SQPN_QP1; 941 } 942 943 mlx5_fill_page_array(&qp->buf, 944 (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas)); 945 946 err = mlx5_db_alloc(dev->mdev, &qp->db); 947 if (err) { 948 mlx5_ib_dbg(dev, "err %d\n", err); 949 goto err_free; 950 } 951 952 qp->sq.wrid = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wrid), GFP_KERNEL); 953 qp->sq.wr_data = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wr_data), GFP_KERNEL); 954 qp->rq.wrid = kmalloc(qp->rq.wqe_cnt * sizeof(*qp->rq.wrid), GFP_KERNEL); 955 qp->sq.w_list = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.w_list), GFP_KERNEL); 956 qp->sq.wqe_head = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wqe_head), GFP_KERNEL); 957 958 if (!qp->sq.wrid || !qp->sq.wr_data || !qp->rq.wrid || 959 !qp->sq.w_list || !qp->sq.wqe_head) { 960 err = -ENOMEM; 961 goto err_wrid; 962 } 963 qp->create_type = MLX5_QP_KERNEL; 964 965 return 0; 966 967 err_wrid: 968 mlx5_db_free(dev->mdev, &qp->db); 969 kfree(qp->sq.wqe_head); 970 kfree(qp->sq.w_list); 971 kfree(qp->sq.wrid); 972 kfree(qp->sq.wr_data); 973 kfree(qp->rq.wrid); 974 975 err_free: 976 kvfree(*in); 977 978 err_buf: 979 mlx5_buf_free(dev->mdev, &qp->buf); 980 981 err_uuar: 982 free_uuar(&dev->mdev->priv.uuari, uuarn); 983 return err; 984 } 985 986 static void destroy_qp_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp) 987 { 988 mlx5_db_free(dev->mdev, &qp->db); 989 kfree(qp->sq.wqe_head); 990 kfree(qp->sq.w_list); 991 kfree(qp->sq.wrid); 992 kfree(qp->sq.wr_data); 993 kfree(qp->rq.wrid); 994 mlx5_buf_free(dev->mdev, &qp->buf); 995 free_uuar(&dev->mdev->priv.uuari, qp->bf->uuarn); 996 } 997 998 static u32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr) 999 { 1000 if (attr->srq || (attr->qp_type == IB_QPT_XRC_TGT) || 1001 (attr->qp_type == IB_QPT_XRC_INI)) 1002 return MLX5_SRQ_RQ; 1003 else if (!qp->has_rq) 1004 return MLX5_ZERO_LEN_RQ; 1005 else 1006 return MLX5_NON_ZERO_RQ; 1007 } 1008 1009 static int is_connected(enum ib_qp_type qp_type) 1010 { 1011 if (qp_type == IB_QPT_RC || qp_type == IB_QPT_UC) 1012 return 1; 1013 1014 return 0; 1015 } 1016 1017 static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev, 1018 struct mlx5_ib_sq *sq, u32 tdn) 1019 { 1020 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0}; 1021 void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); 1022 1023 MLX5_SET(tisc, tisc, transport_domain, tdn); 1024 return mlx5_core_create_tis(dev->mdev, in, sizeof(in), &sq->tisn); 1025 } 1026 1027 static void destroy_raw_packet_qp_tis(struct mlx5_ib_dev *dev, 1028 struct mlx5_ib_sq *sq) 1029 { 1030 mlx5_core_destroy_tis(dev->mdev, sq->tisn); 1031 } 1032 1033 static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev, 1034 struct mlx5_ib_sq *sq, void *qpin, 1035 struct ib_pd *pd) 1036 { 1037 struct mlx5_ib_ubuffer *ubuffer = &sq->ubuffer; 1038 __be64 *pas; 1039 void *in; 1040 void *sqc; 1041 void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc); 1042 void *wq; 1043 int inlen; 1044 int err; 1045 int page_shift = 0; 1046 int npages; 1047 int ncont = 0; 1048 u32 offset = 0; 1049 1050 err = mlx5_ib_umem_get(dev, pd, ubuffer->buf_addr, ubuffer->buf_size, 1051 &sq->ubuffer.umem, &npages, &page_shift, 1052 &ncont, &offset); 1053 if (err) 1054 return err; 1055 1056 inlen = MLX5_ST_SZ_BYTES(create_sq_in) + sizeof(u64) * ncont; 1057 in = mlx5_vzalloc(inlen); 1058 if (!in) { 1059 err = -ENOMEM; 1060 goto err_umem; 1061 } 1062 1063 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); 1064 MLX5_SET(sqc, sqc, flush_in_error_en, 1); 1065 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); 1066 MLX5_SET(sqc, sqc, user_index, MLX5_GET(qpc, qpc, user_index)); 1067 MLX5_SET(sqc, sqc, cqn, MLX5_GET(qpc, qpc, cqn_snd)); 1068 MLX5_SET(sqc, sqc, tis_lst_sz, 1); 1069 MLX5_SET(sqc, sqc, tis_num_0, sq->tisn); 1070 1071 wq = MLX5_ADDR_OF(sqc, sqc, wq); 1072 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); 1073 MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd)); 1074 MLX5_SET(wq, wq, uar_page, MLX5_GET(qpc, qpc, uar_page)); 1075 MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr)); 1076 MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB)); 1077 MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_sq_size)); 1078 MLX5_SET(wq, wq, log_wq_pg_sz, page_shift - MLX5_ADAPTER_PAGE_SHIFT); 1079 MLX5_SET(wq, wq, page_offset, offset); 1080 1081 pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas); 1082 mlx5_ib_populate_pas(dev, sq->ubuffer.umem, page_shift, pas, 0); 1083 1084 err = mlx5_core_create_sq_tracked(dev->mdev, in, inlen, &sq->base.mqp); 1085 1086 kvfree(in); 1087 1088 if (err) 1089 goto err_umem; 1090 1091 return 0; 1092 1093 err_umem: 1094 ib_umem_release(sq->ubuffer.umem); 1095 sq->ubuffer.umem = NULL; 1096 1097 return err; 1098 } 1099 1100 static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev *dev, 1101 struct mlx5_ib_sq *sq) 1102 { 1103 mlx5_core_destroy_sq_tracked(dev->mdev, &sq->base.mqp); 1104 ib_umem_release(sq->ubuffer.umem); 1105 } 1106 1107 static int get_rq_pas_size(void *qpc) 1108 { 1109 u32 log_page_size = MLX5_GET(qpc, qpc, log_page_size) + 12; 1110 u32 log_rq_stride = MLX5_GET(qpc, qpc, log_rq_stride); 1111 u32 log_rq_size = MLX5_GET(qpc, qpc, log_rq_size); 1112 u32 page_offset = MLX5_GET(qpc, qpc, page_offset); 1113 u32 po_quanta = 1 << (log_page_size - 6); 1114 u32 rq_sz = 1 << (log_rq_size + 4 + log_rq_stride); 1115 u32 page_size = 1 << log_page_size; 1116 u32 rq_sz_po = rq_sz + (page_offset * po_quanta); 1117 u32 rq_num_pas = (rq_sz_po + page_size - 1) / page_size; 1118 1119 return rq_num_pas * sizeof(u64); 1120 } 1121 1122 static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev, 1123 struct mlx5_ib_rq *rq, void *qpin) 1124 { 1125 struct mlx5_ib_qp *mqp = rq->base.container_mibqp; 1126 __be64 *pas; 1127 __be64 *qp_pas; 1128 void *in; 1129 void *rqc; 1130 void *wq; 1131 void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc); 1132 int inlen; 1133 int err; 1134 u32 rq_pas_size = get_rq_pas_size(qpc); 1135 1136 inlen = MLX5_ST_SZ_BYTES(create_rq_in) + rq_pas_size; 1137 in = mlx5_vzalloc(inlen); 1138 if (!in) 1139 return -ENOMEM; 1140 1141 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); 1142 MLX5_SET(rqc, rqc, vsd, 1); 1143 MLX5_SET(rqc, rqc, mem_rq_type, MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE); 1144 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); 1145 MLX5_SET(rqc, rqc, flush_in_error_en, 1); 1146 MLX5_SET(rqc, rqc, user_index, MLX5_GET(qpc, qpc, user_index)); 1147 MLX5_SET(rqc, rqc, cqn, MLX5_GET(qpc, qpc, cqn_rcv)); 1148 1149 if (mqp->flags & MLX5_IB_QP_CAP_SCATTER_FCS) 1150 MLX5_SET(rqc, rqc, scatter_fcs, 1); 1151 1152 wq = MLX5_ADDR_OF(rqc, rqc, wq); 1153 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); 1154 MLX5_SET(wq, wq, end_padding_mode, 1155 MLX5_GET(qpc, qpc, end_padding_mode)); 1156 MLX5_SET(wq, wq, page_offset, MLX5_GET(qpc, qpc, page_offset)); 1157 MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd)); 1158 MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr)); 1159 MLX5_SET(wq, wq, log_wq_stride, MLX5_GET(qpc, qpc, log_rq_stride) + 4); 1160 MLX5_SET(wq, wq, log_wq_pg_sz, MLX5_GET(qpc, qpc, log_page_size)); 1161 MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_rq_size)); 1162 1163 pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas); 1164 qp_pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, qpin, pas); 1165 memcpy(pas, qp_pas, rq_pas_size); 1166 1167 err = mlx5_core_create_rq_tracked(dev->mdev, in, inlen, &rq->base.mqp); 1168 1169 kvfree(in); 1170 1171 return err; 1172 } 1173 1174 static void destroy_raw_packet_qp_rq(struct mlx5_ib_dev *dev, 1175 struct mlx5_ib_rq *rq) 1176 { 1177 mlx5_core_destroy_rq_tracked(dev->mdev, &rq->base.mqp); 1178 } 1179 1180 static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev, 1181 struct mlx5_ib_rq *rq, u32 tdn) 1182 { 1183 u32 *in; 1184 void *tirc; 1185 int inlen; 1186 int err; 1187 1188 inlen = MLX5_ST_SZ_BYTES(create_tir_in); 1189 in = mlx5_vzalloc(inlen); 1190 if (!in) 1191 return -ENOMEM; 1192 1193 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 1194 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT); 1195 MLX5_SET(tirc, tirc, inline_rqn, rq->base.mqp.qpn); 1196 MLX5_SET(tirc, tirc, transport_domain, tdn); 1197 1198 err = mlx5_core_create_tir(dev->mdev, in, inlen, &rq->tirn); 1199 1200 kvfree(in); 1201 1202 return err; 1203 } 1204 1205 static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev, 1206 struct mlx5_ib_rq *rq) 1207 { 1208 mlx5_core_destroy_tir(dev->mdev, rq->tirn); 1209 } 1210 1211 static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, 1212 u32 *in, 1213 struct ib_pd *pd) 1214 { 1215 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp; 1216 struct mlx5_ib_sq *sq = &raw_packet_qp->sq; 1217 struct mlx5_ib_rq *rq = &raw_packet_qp->rq; 1218 struct ib_uobject *uobj = pd->uobject; 1219 struct ib_ucontext *ucontext = uobj->context; 1220 struct mlx5_ib_ucontext *mucontext = to_mucontext(ucontext); 1221 int err; 1222 u32 tdn = mucontext->tdn; 1223 1224 if (qp->sq.wqe_cnt) { 1225 err = create_raw_packet_qp_tis(dev, sq, tdn); 1226 if (err) 1227 return err; 1228 1229 err = create_raw_packet_qp_sq(dev, sq, in, pd); 1230 if (err) 1231 goto err_destroy_tis; 1232 1233 sq->base.container_mibqp = qp; 1234 } 1235 1236 if (qp->rq.wqe_cnt) { 1237 rq->base.container_mibqp = qp; 1238 1239 err = create_raw_packet_qp_rq(dev, rq, in); 1240 if (err) 1241 goto err_destroy_sq; 1242 1243 1244 err = create_raw_packet_qp_tir(dev, rq, tdn); 1245 if (err) 1246 goto err_destroy_rq; 1247 } 1248 1249 qp->trans_qp.base.mqp.qpn = qp->sq.wqe_cnt ? sq->base.mqp.qpn : 1250 rq->base.mqp.qpn; 1251 1252 return 0; 1253 1254 err_destroy_rq: 1255 destroy_raw_packet_qp_rq(dev, rq); 1256 err_destroy_sq: 1257 if (!qp->sq.wqe_cnt) 1258 return err; 1259 destroy_raw_packet_qp_sq(dev, sq); 1260 err_destroy_tis: 1261 destroy_raw_packet_qp_tis(dev, sq); 1262 1263 return err; 1264 } 1265 1266 static void destroy_raw_packet_qp(struct mlx5_ib_dev *dev, 1267 struct mlx5_ib_qp *qp) 1268 { 1269 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp; 1270 struct mlx5_ib_sq *sq = &raw_packet_qp->sq; 1271 struct mlx5_ib_rq *rq = &raw_packet_qp->rq; 1272 1273 if (qp->rq.wqe_cnt) { 1274 destroy_raw_packet_qp_tir(dev, rq); 1275 destroy_raw_packet_qp_rq(dev, rq); 1276 } 1277 1278 if (qp->sq.wqe_cnt) { 1279 destroy_raw_packet_qp_sq(dev, sq); 1280 destroy_raw_packet_qp_tis(dev, sq); 1281 } 1282 } 1283 1284 static void raw_packet_qp_copy_info(struct mlx5_ib_qp *qp, 1285 struct mlx5_ib_raw_packet_qp *raw_packet_qp) 1286 { 1287 struct mlx5_ib_sq *sq = &raw_packet_qp->sq; 1288 struct mlx5_ib_rq *rq = &raw_packet_qp->rq; 1289 1290 sq->sq = &qp->sq; 1291 rq->rq = &qp->rq; 1292 sq->doorbell = &qp->db; 1293 rq->doorbell = &qp->db; 1294 } 1295 1296 static void destroy_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp) 1297 { 1298 mlx5_core_destroy_tir(dev->mdev, qp->rss_qp.tirn); 1299 } 1300 1301 static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, 1302 struct ib_pd *pd, 1303 struct ib_qp_init_attr *init_attr, 1304 struct ib_udata *udata) 1305 { 1306 struct ib_uobject *uobj = pd->uobject; 1307 struct ib_ucontext *ucontext = uobj->context; 1308 struct mlx5_ib_ucontext *mucontext = to_mucontext(ucontext); 1309 struct mlx5_ib_create_qp_resp resp = {}; 1310 int inlen; 1311 int err; 1312 u32 *in; 1313 void *tirc; 1314 void *hfso; 1315 u32 selected_fields = 0; 1316 size_t min_resp_len; 1317 u32 tdn = mucontext->tdn; 1318 struct mlx5_ib_create_qp_rss ucmd = {}; 1319 size_t required_cmd_sz; 1320 1321 if (init_attr->qp_type != IB_QPT_RAW_PACKET) 1322 return -EOPNOTSUPP; 1323 1324 if (init_attr->create_flags || init_attr->send_cq) 1325 return -EINVAL; 1326 1327 min_resp_len = offsetof(typeof(resp), uuar_index) + sizeof(resp.uuar_index); 1328 if (udata->outlen < min_resp_len) 1329 return -EINVAL; 1330 1331 required_cmd_sz = offsetof(typeof(ucmd), reserved1) + sizeof(ucmd.reserved1); 1332 if (udata->inlen < required_cmd_sz) { 1333 mlx5_ib_dbg(dev, "invalid inlen\n"); 1334 return -EINVAL; 1335 } 1336 1337 if (udata->inlen > sizeof(ucmd) && 1338 !ib_is_udata_cleared(udata, sizeof(ucmd), 1339 udata->inlen - sizeof(ucmd))) { 1340 mlx5_ib_dbg(dev, "inlen is not supported\n"); 1341 return -EOPNOTSUPP; 1342 } 1343 1344 if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) { 1345 mlx5_ib_dbg(dev, "copy failed\n"); 1346 return -EFAULT; 1347 } 1348 1349 if (ucmd.comp_mask) { 1350 mlx5_ib_dbg(dev, "invalid comp mask\n"); 1351 return -EOPNOTSUPP; 1352 } 1353 1354 if (memchr_inv(ucmd.reserved, 0, sizeof(ucmd.reserved)) || ucmd.reserved1) { 1355 mlx5_ib_dbg(dev, "invalid reserved\n"); 1356 return -EOPNOTSUPP; 1357 } 1358 1359 err = ib_copy_to_udata(udata, &resp, min_resp_len); 1360 if (err) { 1361 mlx5_ib_dbg(dev, "copy failed\n"); 1362 return -EINVAL; 1363 } 1364 1365 inlen = MLX5_ST_SZ_BYTES(create_tir_in); 1366 in = mlx5_vzalloc(inlen); 1367 if (!in) 1368 return -ENOMEM; 1369 1370 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 1371 MLX5_SET(tirc, tirc, disp_type, 1372 MLX5_TIRC_DISP_TYPE_INDIRECT); 1373 MLX5_SET(tirc, tirc, indirect_table, 1374 init_attr->rwq_ind_tbl->ind_tbl_num); 1375 MLX5_SET(tirc, tirc, transport_domain, tdn); 1376 1377 hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer); 1378 switch (ucmd.rx_hash_function) { 1379 case MLX5_RX_HASH_FUNC_TOEPLITZ: 1380 { 1381 void *rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key); 1382 size_t len = MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key); 1383 1384 if (len != ucmd.rx_key_len) { 1385 err = -EINVAL; 1386 goto err; 1387 } 1388 1389 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ); 1390 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); 1391 memcpy(rss_key, ucmd.rx_hash_key, len); 1392 break; 1393 } 1394 default: 1395 err = -EOPNOTSUPP; 1396 goto err; 1397 } 1398 1399 if (!ucmd.rx_hash_fields_mask) { 1400 /* special case when this TIR serves as steering entry without hashing */ 1401 if (!init_attr->rwq_ind_tbl->log_ind_tbl_size) 1402 goto create_tir; 1403 err = -EINVAL; 1404 goto err; 1405 } 1406 1407 if (((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) || 1408 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4)) && 1409 ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) || 1410 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))) { 1411 err = -EINVAL; 1412 goto err; 1413 } 1414 1415 /* If none of IPV4 & IPV6 SRC/DST was set - this bit field is ignored */ 1416 if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) || 1417 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4)) 1418 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, 1419 MLX5_L3_PROT_TYPE_IPV4); 1420 else if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) || 1421 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6)) 1422 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, 1423 MLX5_L3_PROT_TYPE_IPV6); 1424 1425 if (((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) || 1426 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP)) && 1427 ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) || 1428 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))) { 1429 err = -EINVAL; 1430 goto err; 1431 } 1432 1433 /* If none of TCP & UDP SRC/DST was set - this bit field is ignored */ 1434 if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) || 1435 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP)) 1436 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, 1437 MLX5_L4_PROT_TYPE_TCP); 1438 else if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) || 1439 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP)) 1440 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, 1441 MLX5_L4_PROT_TYPE_UDP); 1442 1443 if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) || 1444 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6)) 1445 selected_fields |= MLX5_HASH_FIELD_SEL_SRC_IP; 1446 1447 if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4) || 1448 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6)) 1449 selected_fields |= MLX5_HASH_FIELD_SEL_DST_IP; 1450 1451 if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) || 1452 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP)) 1453 selected_fields |= MLX5_HASH_FIELD_SEL_L4_SPORT; 1454 1455 if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP) || 1456 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP)) 1457 selected_fields |= MLX5_HASH_FIELD_SEL_L4_DPORT; 1458 1459 MLX5_SET(rx_hash_field_select, hfso, selected_fields, selected_fields); 1460 1461 create_tir: 1462 err = mlx5_core_create_tir(dev->mdev, in, inlen, &qp->rss_qp.tirn); 1463 1464 if (err) 1465 goto err; 1466 1467 kvfree(in); 1468 /* qpn is reserved for that QP */ 1469 qp->trans_qp.base.mqp.qpn = 0; 1470 qp->flags |= MLX5_IB_QP_RSS; 1471 return 0; 1472 1473 err: 1474 kvfree(in); 1475 return err; 1476 } 1477 1478 static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd, 1479 struct ib_qp_init_attr *init_attr, 1480 struct ib_udata *udata, struct mlx5_ib_qp *qp) 1481 { 1482 struct mlx5_ib_resources *devr = &dev->devr; 1483 int inlen = MLX5_ST_SZ_BYTES(create_qp_in); 1484 struct mlx5_core_dev *mdev = dev->mdev; 1485 struct mlx5_ib_create_qp_resp resp; 1486 struct mlx5_ib_cq *send_cq; 1487 struct mlx5_ib_cq *recv_cq; 1488 unsigned long flags; 1489 u32 uidx = MLX5_IB_DEFAULT_UIDX; 1490 struct mlx5_ib_create_qp ucmd; 1491 struct mlx5_ib_qp_base *base; 1492 void *qpc; 1493 u32 *in; 1494 int err; 1495 1496 base = init_attr->qp_type == IB_QPT_RAW_PACKET ? 1497 &qp->raw_packet_qp.rq.base : 1498 &qp->trans_qp.base; 1499 1500 if (init_attr->qp_type != IB_QPT_RAW_PACKET) 1501 mlx5_ib_odp_create_qp(qp); 1502 1503 mutex_init(&qp->mutex); 1504 spin_lock_init(&qp->sq.lock); 1505 spin_lock_init(&qp->rq.lock); 1506 1507 if (init_attr->rwq_ind_tbl) { 1508 if (!udata) 1509 return -ENOSYS; 1510 1511 err = create_rss_raw_qp_tir(dev, qp, pd, init_attr, udata); 1512 return err; 1513 } 1514 1515 if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) { 1516 if (!MLX5_CAP_GEN(mdev, block_lb_mc)) { 1517 mlx5_ib_dbg(dev, "block multicast loopback isn't supported\n"); 1518 return -EINVAL; 1519 } else { 1520 qp->flags |= MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK; 1521 } 1522 } 1523 1524 if (init_attr->create_flags & 1525 (IB_QP_CREATE_CROSS_CHANNEL | 1526 IB_QP_CREATE_MANAGED_SEND | 1527 IB_QP_CREATE_MANAGED_RECV)) { 1528 if (!MLX5_CAP_GEN(mdev, cd)) { 1529 mlx5_ib_dbg(dev, "cross-channel isn't supported\n"); 1530 return -EINVAL; 1531 } 1532 if (init_attr->create_flags & IB_QP_CREATE_CROSS_CHANNEL) 1533 qp->flags |= MLX5_IB_QP_CROSS_CHANNEL; 1534 if (init_attr->create_flags & IB_QP_CREATE_MANAGED_SEND) 1535 qp->flags |= MLX5_IB_QP_MANAGED_SEND; 1536 if (init_attr->create_flags & IB_QP_CREATE_MANAGED_RECV) 1537 qp->flags |= MLX5_IB_QP_MANAGED_RECV; 1538 } 1539 1540 if (init_attr->qp_type == IB_QPT_UD && 1541 (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)) 1542 if (!MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) { 1543 mlx5_ib_dbg(dev, "ipoib UD lso qp isn't supported\n"); 1544 return -EOPNOTSUPP; 1545 } 1546 1547 if (init_attr->create_flags & IB_QP_CREATE_SCATTER_FCS) { 1548 if (init_attr->qp_type != IB_QPT_RAW_PACKET) { 1549 mlx5_ib_dbg(dev, "Scatter FCS is supported only for Raw Packet QPs"); 1550 return -EOPNOTSUPP; 1551 } 1552 if (!MLX5_CAP_GEN(dev->mdev, eth_net_offloads) || 1553 !MLX5_CAP_ETH(dev->mdev, scatter_fcs)) { 1554 mlx5_ib_dbg(dev, "Scatter FCS isn't supported\n"); 1555 return -EOPNOTSUPP; 1556 } 1557 qp->flags |= MLX5_IB_QP_CAP_SCATTER_FCS; 1558 } 1559 1560 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) 1561 qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE; 1562 1563 if (pd && pd->uobject) { 1564 if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) { 1565 mlx5_ib_dbg(dev, "copy failed\n"); 1566 return -EFAULT; 1567 } 1568 1569 err = get_qp_user_index(to_mucontext(pd->uobject->context), 1570 &ucmd, udata->inlen, &uidx); 1571 if (err) 1572 return err; 1573 1574 qp->wq_sig = !!(ucmd.flags & MLX5_QP_FLAG_SIGNATURE); 1575 qp->scat_cqe = !!(ucmd.flags & MLX5_QP_FLAG_SCATTER_CQE); 1576 } else { 1577 qp->wq_sig = !!wq_signature; 1578 } 1579 1580 qp->has_rq = qp_has_rq(init_attr); 1581 err = set_rq_size(dev, &init_attr->cap, qp->has_rq, 1582 qp, (pd && pd->uobject) ? &ucmd : NULL); 1583 if (err) { 1584 mlx5_ib_dbg(dev, "err %d\n", err); 1585 return err; 1586 } 1587 1588 if (pd) { 1589 if (pd->uobject) { 1590 __u32 max_wqes = 1591 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz); 1592 mlx5_ib_dbg(dev, "requested sq_wqe_count (%d)\n", ucmd.sq_wqe_count); 1593 if (ucmd.rq_wqe_shift != qp->rq.wqe_shift || 1594 ucmd.rq_wqe_count != qp->rq.wqe_cnt) { 1595 mlx5_ib_dbg(dev, "invalid rq params\n"); 1596 return -EINVAL; 1597 } 1598 if (ucmd.sq_wqe_count > max_wqes) { 1599 mlx5_ib_dbg(dev, "requested sq_wqe_count (%d) > max allowed (%d)\n", 1600 ucmd.sq_wqe_count, max_wqes); 1601 return -EINVAL; 1602 } 1603 if (init_attr->create_flags & 1604 mlx5_ib_create_qp_sqpn_qp1()) { 1605 mlx5_ib_dbg(dev, "user-space is not allowed to create UD QPs spoofing as QP1\n"); 1606 return -EINVAL; 1607 } 1608 err = create_user_qp(dev, pd, qp, udata, init_attr, &in, 1609 &resp, &inlen, base); 1610 if (err) 1611 mlx5_ib_dbg(dev, "err %d\n", err); 1612 } else { 1613 err = create_kernel_qp(dev, init_attr, qp, &in, &inlen, 1614 base); 1615 if (err) 1616 mlx5_ib_dbg(dev, "err %d\n", err); 1617 } 1618 1619 if (err) 1620 return err; 1621 } else { 1622 in = mlx5_vzalloc(inlen); 1623 if (!in) 1624 return -ENOMEM; 1625 1626 qp->create_type = MLX5_QP_EMPTY; 1627 } 1628 1629 if (is_sqp(init_attr->qp_type)) 1630 qp->port = init_attr->port_num; 1631 1632 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 1633 1634 MLX5_SET(qpc, qpc, st, to_mlx5_st(init_attr->qp_type)); 1635 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 1636 1637 if (init_attr->qp_type != MLX5_IB_QPT_REG_UMR) 1638 MLX5_SET(qpc, qpc, pd, to_mpd(pd ? pd : devr->p0)->pdn); 1639 else 1640 MLX5_SET(qpc, qpc, latency_sensitive, 1); 1641 1642 1643 if (qp->wq_sig) 1644 MLX5_SET(qpc, qpc, wq_signature, 1); 1645 1646 if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK) 1647 MLX5_SET(qpc, qpc, block_lb_mc, 1); 1648 1649 if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL) 1650 MLX5_SET(qpc, qpc, cd_master, 1); 1651 if (qp->flags & MLX5_IB_QP_MANAGED_SEND) 1652 MLX5_SET(qpc, qpc, cd_slave_send, 1); 1653 if (qp->flags & MLX5_IB_QP_MANAGED_RECV) 1654 MLX5_SET(qpc, qpc, cd_slave_receive, 1); 1655 1656 if (qp->scat_cqe && is_connected(init_attr->qp_type)) { 1657 int rcqe_sz; 1658 int scqe_sz; 1659 1660 rcqe_sz = mlx5_ib_get_cqe_size(dev, init_attr->recv_cq); 1661 scqe_sz = mlx5_ib_get_cqe_size(dev, init_attr->send_cq); 1662 1663 if (rcqe_sz == 128) 1664 MLX5_SET(qpc, qpc, cs_res, MLX5_RES_SCAT_DATA64_CQE); 1665 else 1666 MLX5_SET(qpc, qpc, cs_res, MLX5_RES_SCAT_DATA32_CQE); 1667 1668 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) { 1669 if (scqe_sz == 128) 1670 MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA64_CQE); 1671 else 1672 MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA32_CQE); 1673 } 1674 } 1675 1676 if (qp->rq.wqe_cnt) { 1677 MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4); 1678 MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt)); 1679 } 1680 1681 MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr)); 1682 1683 if (qp->sq.wqe_cnt) 1684 MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt)); 1685 else 1686 MLX5_SET(qpc, qpc, no_sq, 1); 1687 1688 /* Set default resources */ 1689 switch (init_attr->qp_type) { 1690 case IB_QPT_XRC_TGT: 1691 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn); 1692 MLX5_SET(qpc, qpc, cqn_snd, to_mcq(devr->c0)->mcq.cqn); 1693 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn); 1694 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(init_attr->xrcd)->xrcdn); 1695 break; 1696 case IB_QPT_XRC_INI: 1697 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn); 1698 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn); 1699 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn); 1700 break; 1701 default: 1702 if (init_attr->srq) { 1703 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x0)->xrcdn); 1704 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(init_attr->srq)->msrq.srqn); 1705 } else { 1706 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn); 1707 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s1)->msrq.srqn); 1708 } 1709 } 1710 1711 if (init_attr->send_cq) 1712 MLX5_SET(qpc, qpc, cqn_snd, to_mcq(init_attr->send_cq)->mcq.cqn); 1713 1714 if (init_attr->recv_cq) 1715 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(init_attr->recv_cq)->mcq.cqn); 1716 1717 MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma); 1718 1719 /* 0xffffff means we ask to work with cqe version 0 */ 1720 if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1) 1721 MLX5_SET(qpc, qpc, user_index, uidx); 1722 1723 /* we use IB_QP_CREATE_IPOIB_UD_LSO to indicates ipoib qp */ 1724 if (init_attr->qp_type == IB_QPT_UD && 1725 (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)) { 1726 MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1); 1727 qp->flags |= MLX5_IB_QP_LSO; 1728 } 1729 1730 if (init_attr->qp_type == IB_QPT_RAW_PACKET) { 1731 qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd.sq_buf_addr; 1732 raw_packet_qp_copy_info(qp, &qp->raw_packet_qp); 1733 err = create_raw_packet_qp(dev, qp, in, pd); 1734 } else { 1735 err = mlx5_core_create_qp(dev->mdev, &base->mqp, in, inlen); 1736 } 1737 1738 if (err) { 1739 mlx5_ib_dbg(dev, "create qp failed\n"); 1740 goto err_create; 1741 } 1742 1743 kvfree(in); 1744 1745 base->container_mibqp = qp; 1746 base->mqp.event = mlx5_ib_qp_event; 1747 1748 get_cqs(init_attr->qp_type, init_attr->send_cq, init_attr->recv_cq, 1749 &send_cq, &recv_cq); 1750 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags); 1751 mlx5_ib_lock_cqs(send_cq, recv_cq); 1752 /* Maintain device to QPs access, needed for further handling via reset 1753 * flow 1754 */ 1755 list_add_tail(&qp->qps_list, &dev->qp_list); 1756 /* Maintain CQ to QPs access, needed for further handling via reset flow 1757 */ 1758 if (send_cq) 1759 list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp); 1760 if (recv_cq) 1761 list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp); 1762 mlx5_ib_unlock_cqs(send_cq, recv_cq); 1763 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags); 1764 1765 return 0; 1766 1767 err_create: 1768 if (qp->create_type == MLX5_QP_USER) 1769 destroy_qp_user(pd, qp, base); 1770 else if (qp->create_type == MLX5_QP_KERNEL) 1771 destroy_qp_kernel(dev, qp); 1772 1773 kvfree(in); 1774 return err; 1775 } 1776 1777 static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq) 1778 __acquires(&send_cq->lock) __acquires(&recv_cq->lock) 1779 { 1780 if (send_cq) { 1781 if (recv_cq) { 1782 if (send_cq->mcq.cqn < recv_cq->mcq.cqn) { 1783 spin_lock(&send_cq->lock); 1784 spin_lock_nested(&recv_cq->lock, 1785 SINGLE_DEPTH_NESTING); 1786 } else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) { 1787 spin_lock(&send_cq->lock); 1788 __acquire(&recv_cq->lock); 1789 } else { 1790 spin_lock(&recv_cq->lock); 1791 spin_lock_nested(&send_cq->lock, 1792 SINGLE_DEPTH_NESTING); 1793 } 1794 } else { 1795 spin_lock(&send_cq->lock); 1796 __acquire(&recv_cq->lock); 1797 } 1798 } else if (recv_cq) { 1799 spin_lock(&recv_cq->lock); 1800 __acquire(&send_cq->lock); 1801 } else { 1802 __acquire(&send_cq->lock); 1803 __acquire(&recv_cq->lock); 1804 } 1805 } 1806 1807 static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq) 1808 __releases(&send_cq->lock) __releases(&recv_cq->lock) 1809 { 1810 if (send_cq) { 1811 if (recv_cq) { 1812 if (send_cq->mcq.cqn < recv_cq->mcq.cqn) { 1813 spin_unlock(&recv_cq->lock); 1814 spin_unlock(&send_cq->lock); 1815 } else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) { 1816 __release(&recv_cq->lock); 1817 spin_unlock(&send_cq->lock); 1818 } else { 1819 spin_unlock(&send_cq->lock); 1820 spin_unlock(&recv_cq->lock); 1821 } 1822 } else { 1823 __release(&recv_cq->lock); 1824 spin_unlock(&send_cq->lock); 1825 } 1826 } else if (recv_cq) { 1827 __release(&send_cq->lock); 1828 spin_unlock(&recv_cq->lock); 1829 } else { 1830 __release(&recv_cq->lock); 1831 __release(&send_cq->lock); 1832 } 1833 } 1834 1835 static struct mlx5_ib_pd *get_pd(struct mlx5_ib_qp *qp) 1836 { 1837 return to_mpd(qp->ibqp.pd); 1838 } 1839 1840 static void get_cqs(enum ib_qp_type qp_type, 1841 struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq, 1842 struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq) 1843 { 1844 switch (qp_type) { 1845 case IB_QPT_XRC_TGT: 1846 *send_cq = NULL; 1847 *recv_cq = NULL; 1848 break; 1849 case MLX5_IB_QPT_REG_UMR: 1850 case IB_QPT_XRC_INI: 1851 *send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL; 1852 *recv_cq = NULL; 1853 break; 1854 1855 case IB_QPT_SMI: 1856 case MLX5_IB_QPT_HW_GSI: 1857 case IB_QPT_RC: 1858 case IB_QPT_UC: 1859 case IB_QPT_UD: 1860 case IB_QPT_RAW_IPV6: 1861 case IB_QPT_RAW_ETHERTYPE: 1862 case IB_QPT_RAW_PACKET: 1863 *send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL; 1864 *recv_cq = ib_recv_cq ? to_mcq(ib_recv_cq) : NULL; 1865 break; 1866 1867 case IB_QPT_MAX: 1868 default: 1869 *send_cq = NULL; 1870 *recv_cq = NULL; 1871 break; 1872 } 1873 } 1874 1875 static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, 1876 const struct mlx5_modify_raw_qp_param *raw_qp_param, 1877 u8 lag_tx_affinity); 1878 1879 static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp) 1880 { 1881 struct mlx5_ib_cq *send_cq, *recv_cq; 1882 struct mlx5_ib_qp_base *base = &qp->trans_qp.base; 1883 unsigned long flags; 1884 int err; 1885 1886 if (qp->ibqp.rwq_ind_tbl) { 1887 destroy_rss_raw_qp_tir(dev, qp); 1888 return; 1889 } 1890 1891 base = qp->ibqp.qp_type == IB_QPT_RAW_PACKET ? 1892 &qp->raw_packet_qp.rq.base : 1893 &qp->trans_qp.base; 1894 1895 if (qp->state != IB_QPS_RESET) { 1896 if (qp->ibqp.qp_type != IB_QPT_RAW_PACKET) { 1897 mlx5_ib_qp_disable_pagefaults(qp); 1898 err = mlx5_core_qp_modify(dev->mdev, 1899 MLX5_CMD_OP_2RST_QP, 0, 1900 NULL, &base->mqp); 1901 } else { 1902 struct mlx5_modify_raw_qp_param raw_qp_param = { 1903 .operation = MLX5_CMD_OP_2RST_QP 1904 }; 1905 1906 err = modify_raw_packet_qp(dev, qp, &raw_qp_param, 0); 1907 } 1908 if (err) 1909 mlx5_ib_warn(dev, "mlx5_ib: modify QP 0x%06x to RESET failed\n", 1910 base->mqp.qpn); 1911 } 1912 1913 get_cqs(qp->ibqp.qp_type, qp->ibqp.send_cq, qp->ibqp.recv_cq, 1914 &send_cq, &recv_cq); 1915 1916 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags); 1917 mlx5_ib_lock_cqs(send_cq, recv_cq); 1918 /* del from lists under both locks above to protect reset flow paths */ 1919 list_del(&qp->qps_list); 1920 if (send_cq) 1921 list_del(&qp->cq_send_list); 1922 1923 if (recv_cq) 1924 list_del(&qp->cq_recv_list); 1925 1926 if (qp->create_type == MLX5_QP_KERNEL) { 1927 __mlx5_ib_cq_clean(recv_cq, base->mqp.qpn, 1928 qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL); 1929 if (send_cq != recv_cq) 1930 __mlx5_ib_cq_clean(send_cq, base->mqp.qpn, 1931 NULL); 1932 } 1933 mlx5_ib_unlock_cqs(send_cq, recv_cq); 1934 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags); 1935 1936 if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET) { 1937 destroy_raw_packet_qp(dev, qp); 1938 } else { 1939 err = mlx5_core_destroy_qp(dev->mdev, &base->mqp); 1940 if (err) 1941 mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n", 1942 base->mqp.qpn); 1943 } 1944 1945 if (qp->create_type == MLX5_QP_KERNEL) 1946 destroy_qp_kernel(dev, qp); 1947 else if (qp->create_type == MLX5_QP_USER) 1948 destroy_qp_user(&get_pd(qp)->ibpd, qp, base); 1949 } 1950 1951 static const char *ib_qp_type_str(enum ib_qp_type type) 1952 { 1953 switch (type) { 1954 case IB_QPT_SMI: 1955 return "IB_QPT_SMI"; 1956 case IB_QPT_GSI: 1957 return "IB_QPT_GSI"; 1958 case IB_QPT_RC: 1959 return "IB_QPT_RC"; 1960 case IB_QPT_UC: 1961 return "IB_QPT_UC"; 1962 case IB_QPT_UD: 1963 return "IB_QPT_UD"; 1964 case IB_QPT_RAW_IPV6: 1965 return "IB_QPT_RAW_IPV6"; 1966 case IB_QPT_RAW_ETHERTYPE: 1967 return "IB_QPT_RAW_ETHERTYPE"; 1968 case IB_QPT_XRC_INI: 1969 return "IB_QPT_XRC_INI"; 1970 case IB_QPT_XRC_TGT: 1971 return "IB_QPT_XRC_TGT"; 1972 case IB_QPT_RAW_PACKET: 1973 return "IB_QPT_RAW_PACKET"; 1974 case MLX5_IB_QPT_REG_UMR: 1975 return "MLX5_IB_QPT_REG_UMR"; 1976 case IB_QPT_MAX: 1977 default: 1978 return "Invalid QP type"; 1979 } 1980 } 1981 1982 struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd, 1983 struct ib_qp_init_attr *init_attr, 1984 struct ib_udata *udata) 1985 { 1986 struct mlx5_ib_dev *dev; 1987 struct mlx5_ib_qp *qp; 1988 u16 xrcdn = 0; 1989 int err; 1990 1991 if (pd) { 1992 dev = to_mdev(pd->device); 1993 1994 if (init_attr->qp_type == IB_QPT_RAW_PACKET) { 1995 if (!pd->uobject) { 1996 mlx5_ib_dbg(dev, "Raw Packet QP is not supported for kernel consumers\n"); 1997 return ERR_PTR(-EINVAL); 1998 } else if (!to_mucontext(pd->uobject->context)->cqe_version) { 1999 mlx5_ib_dbg(dev, "Raw Packet QP is only supported for CQE version > 0\n"); 2000 return ERR_PTR(-EINVAL); 2001 } 2002 } 2003 } else { 2004 /* being cautious here */ 2005 if (init_attr->qp_type != IB_QPT_XRC_TGT && 2006 init_attr->qp_type != MLX5_IB_QPT_REG_UMR) { 2007 pr_warn("%s: no PD for transport %s\n", __func__, 2008 ib_qp_type_str(init_attr->qp_type)); 2009 return ERR_PTR(-EINVAL); 2010 } 2011 dev = to_mdev(to_mxrcd(init_attr->xrcd)->ibxrcd.device); 2012 } 2013 2014 switch (init_attr->qp_type) { 2015 case IB_QPT_XRC_TGT: 2016 case IB_QPT_XRC_INI: 2017 if (!MLX5_CAP_GEN(dev->mdev, xrc)) { 2018 mlx5_ib_dbg(dev, "XRC not supported\n"); 2019 return ERR_PTR(-ENOSYS); 2020 } 2021 init_attr->recv_cq = NULL; 2022 if (init_attr->qp_type == IB_QPT_XRC_TGT) { 2023 xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn; 2024 init_attr->send_cq = NULL; 2025 } 2026 2027 /* fall through */ 2028 case IB_QPT_RAW_PACKET: 2029 case IB_QPT_RC: 2030 case IB_QPT_UC: 2031 case IB_QPT_UD: 2032 case IB_QPT_SMI: 2033 case MLX5_IB_QPT_HW_GSI: 2034 case MLX5_IB_QPT_REG_UMR: 2035 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 2036 if (!qp) 2037 return ERR_PTR(-ENOMEM); 2038 2039 err = create_qp_common(dev, pd, init_attr, udata, qp); 2040 if (err) { 2041 mlx5_ib_dbg(dev, "create_qp_common failed\n"); 2042 kfree(qp); 2043 return ERR_PTR(err); 2044 } 2045 2046 if (is_qp0(init_attr->qp_type)) 2047 qp->ibqp.qp_num = 0; 2048 else if (is_qp1(init_attr->qp_type)) 2049 qp->ibqp.qp_num = 1; 2050 else 2051 qp->ibqp.qp_num = qp->trans_qp.base.mqp.qpn; 2052 2053 mlx5_ib_dbg(dev, "ib qpnum 0x%x, mlx qpn 0x%x, rcqn 0x%x, scqn 0x%x\n", 2054 qp->ibqp.qp_num, qp->trans_qp.base.mqp.qpn, 2055 to_mcq(init_attr->recv_cq)->mcq.cqn, 2056 to_mcq(init_attr->send_cq)->mcq.cqn); 2057 2058 qp->trans_qp.xrcdn = xrcdn; 2059 2060 break; 2061 2062 case IB_QPT_GSI: 2063 return mlx5_ib_gsi_create_qp(pd, init_attr); 2064 2065 case IB_QPT_RAW_IPV6: 2066 case IB_QPT_RAW_ETHERTYPE: 2067 case IB_QPT_MAX: 2068 default: 2069 mlx5_ib_dbg(dev, "unsupported qp type %d\n", 2070 init_attr->qp_type); 2071 /* Don't support raw QPs */ 2072 return ERR_PTR(-EINVAL); 2073 } 2074 2075 return &qp->ibqp; 2076 } 2077 2078 int mlx5_ib_destroy_qp(struct ib_qp *qp) 2079 { 2080 struct mlx5_ib_dev *dev = to_mdev(qp->device); 2081 struct mlx5_ib_qp *mqp = to_mqp(qp); 2082 2083 if (unlikely(qp->qp_type == IB_QPT_GSI)) 2084 return mlx5_ib_gsi_destroy_qp(qp); 2085 2086 destroy_qp_common(dev, mqp); 2087 2088 kfree(mqp); 2089 2090 return 0; 2091 } 2092 2093 static __be32 to_mlx5_access_flags(struct mlx5_ib_qp *qp, const struct ib_qp_attr *attr, 2094 int attr_mask) 2095 { 2096 u32 hw_access_flags = 0; 2097 u8 dest_rd_atomic; 2098 u32 access_flags; 2099 2100 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) 2101 dest_rd_atomic = attr->max_dest_rd_atomic; 2102 else 2103 dest_rd_atomic = qp->trans_qp.resp_depth; 2104 2105 if (attr_mask & IB_QP_ACCESS_FLAGS) 2106 access_flags = attr->qp_access_flags; 2107 else 2108 access_flags = qp->trans_qp.atomic_rd_en; 2109 2110 if (!dest_rd_atomic) 2111 access_flags &= IB_ACCESS_REMOTE_WRITE; 2112 2113 if (access_flags & IB_ACCESS_REMOTE_READ) 2114 hw_access_flags |= MLX5_QP_BIT_RRE; 2115 if (access_flags & IB_ACCESS_REMOTE_ATOMIC) 2116 hw_access_flags |= (MLX5_QP_BIT_RAE | MLX5_ATOMIC_MODE_CX); 2117 if (access_flags & IB_ACCESS_REMOTE_WRITE) 2118 hw_access_flags |= MLX5_QP_BIT_RWE; 2119 2120 return cpu_to_be32(hw_access_flags); 2121 } 2122 2123 enum { 2124 MLX5_PATH_FLAG_FL = 1 << 0, 2125 MLX5_PATH_FLAG_FREE_AR = 1 << 1, 2126 MLX5_PATH_FLAG_COUNTER = 1 << 2, 2127 }; 2128 2129 static int ib_rate_to_mlx5(struct mlx5_ib_dev *dev, u8 rate) 2130 { 2131 if (rate == IB_RATE_PORT_CURRENT) { 2132 return 0; 2133 } else if (rate < IB_RATE_2_5_GBPS || rate > IB_RATE_300_GBPS) { 2134 return -EINVAL; 2135 } else { 2136 while (rate != IB_RATE_2_5_GBPS && 2137 !(1 << (rate + MLX5_STAT_RATE_OFFSET) & 2138 MLX5_CAP_GEN(dev->mdev, stat_rate_support))) 2139 --rate; 2140 } 2141 2142 return rate + MLX5_STAT_RATE_OFFSET; 2143 } 2144 2145 static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev, 2146 struct mlx5_ib_sq *sq, u8 sl) 2147 { 2148 void *in; 2149 void *tisc; 2150 int inlen; 2151 int err; 2152 2153 inlen = MLX5_ST_SZ_BYTES(modify_tis_in); 2154 in = mlx5_vzalloc(inlen); 2155 if (!in) 2156 return -ENOMEM; 2157 2158 MLX5_SET(modify_tis_in, in, bitmask.prio, 1); 2159 2160 tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx); 2161 MLX5_SET(tisc, tisc, prio, ((sl & 0x7) << 1)); 2162 2163 err = mlx5_core_modify_tis(dev, sq->tisn, in, inlen); 2164 2165 kvfree(in); 2166 2167 return err; 2168 } 2169 2170 static int modify_raw_packet_tx_affinity(struct mlx5_core_dev *dev, 2171 struct mlx5_ib_sq *sq, u8 tx_affinity) 2172 { 2173 void *in; 2174 void *tisc; 2175 int inlen; 2176 int err; 2177 2178 inlen = MLX5_ST_SZ_BYTES(modify_tis_in); 2179 in = mlx5_vzalloc(inlen); 2180 if (!in) 2181 return -ENOMEM; 2182 2183 MLX5_SET(modify_tis_in, in, bitmask.lag_tx_port_affinity, 1); 2184 2185 tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx); 2186 MLX5_SET(tisc, tisc, lag_tx_port_affinity, tx_affinity); 2187 2188 err = mlx5_core_modify_tis(dev, sq->tisn, in, inlen); 2189 2190 kvfree(in); 2191 2192 return err; 2193 } 2194 2195 static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, 2196 const struct ib_ah_attr *ah, 2197 struct mlx5_qp_path *path, u8 port, int attr_mask, 2198 u32 path_flags, const struct ib_qp_attr *attr, 2199 bool alt) 2200 { 2201 enum rdma_link_layer ll = rdma_port_get_link_layer(&dev->ib_dev, port); 2202 int err; 2203 2204 if (attr_mask & IB_QP_PKEY_INDEX) 2205 path->pkey_index = cpu_to_be16(alt ? attr->alt_pkey_index : 2206 attr->pkey_index); 2207 2208 if (ah->ah_flags & IB_AH_GRH) { 2209 if (ah->grh.sgid_index >= 2210 dev->mdev->port_caps[port - 1].gid_table_len) { 2211 pr_err("sgid_index (%u) too large. max is %d\n", 2212 ah->grh.sgid_index, 2213 dev->mdev->port_caps[port - 1].gid_table_len); 2214 return -EINVAL; 2215 } 2216 } 2217 2218 if (ll == IB_LINK_LAYER_ETHERNET) { 2219 if (!(ah->ah_flags & IB_AH_GRH)) 2220 return -EINVAL; 2221 memcpy(path->rmac, ah->dmac, sizeof(ah->dmac)); 2222 path->udp_sport = mlx5_get_roce_udp_sport(dev, port, 2223 ah->grh.sgid_index); 2224 path->dci_cfi_prio_sl = (ah->sl & 0x7) << 4; 2225 } else { 2226 path->fl_free_ar = (path_flags & MLX5_PATH_FLAG_FL) ? 0x80 : 0; 2227 path->fl_free_ar |= 2228 (path_flags & MLX5_PATH_FLAG_FREE_AR) ? 0x40 : 0; 2229 path->rlid = cpu_to_be16(ah->dlid); 2230 path->grh_mlid = ah->src_path_bits & 0x7f; 2231 if (ah->ah_flags & IB_AH_GRH) 2232 path->grh_mlid |= 1 << 7; 2233 path->dci_cfi_prio_sl = ah->sl & 0xf; 2234 } 2235 2236 if (ah->ah_flags & IB_AH_GRH) { 2237 path->mgid_index = ah->grh.sgid_index; 2238 path->hop_limit = ah->grh.hop_limit; 2239 path->tclass_flowlabel = 2240 cpu_to_be32((ah->grh.traffic_class << 20) | 2241 (ah->grh.flow_label)); 2242 memcpy(path->rgid, ah->grh.dgid.raw, 16); 2243 } 2244 2245 err = ib_rate_to_mlx5(dev, ah->static_rate); 2246 if (err < 0) 2247 return err; 2248 path->static_rate = err; 2249 path->port = port; 2250 2251 if (attr_mask & IB_QP_TIMEOUT) 2252 path->ackto_lt = (alt ? attr->alt_timeout : attr->timeout) << 3; 2253 2254 if ((qp->ibqp.qp_type == IB_QPT_RAW_PACKET) && qp->sq.wqe_cnt) 2255 return modify_raw_packet_eth_prio(dev->mdev, 2256 &qp->raw_packet_qp.sq, 2257 ah->sl & 0xf); 2258 2259 return 0; 2260 } 2261 2262 static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_QP_ST_MAX] = { 2263 [MLX5_QP_STATE_INIT] = { 2264 [MLX5_QP_STATE_INIT] = { 2265 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE | 2266 MLX5_QP_OPTPAR_RAE | 2267 MLX5_QP_OPTPAR_RWE | 2268 MLX5_QP_OPTPAR_PKEY_INDEX | 2269 MLX5_QP_OPTPAR_PRI_PORT, 2270 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE | 2271 MLX5_QP_OPTPAR_PKEY_INDEX | 2272 MLX5_QP_OPTPAR_PRI_PORT, 2273 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX | 2274 MLX5_QP_OPTPAR_Q_KEY | 2275 MLX5_QP_OPTPAR_PRI_PORT, 2276 }, 2277 [MLX5_QP_STATE_RTR] = { 2278 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH | 2279 MLX5_QP_OPTPAR_RRE | 2280 MLX5_QP_OPTPAR_RAE | 2281 MLX5_QP_OPTPAR_RWE | 2282 MLX5_QP_OPTPAR_PKEY_INDEX, 2283 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH | 2284 MLX5_QP_OPTPAR_RWE | 2285 MLX5_QP_OPTPAR_PKEY_INDEX, 2286 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX | 2287 MLX5_QP_OPTPAR_Q_KEY, 2288 [MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_PKEY_INDEX | 2289 MLX5_QP_OPTPAR_Q_KEY, 2290 [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH | 2291 MLX5_QP_OPTPAR_RRE | 2292 MLX5_QP_OPTPAR_RAE | 2293 MLX5_QP_OPTPAR_RWE | 2294 MLX5_QP_OPTPAR_PKEY_INDEX, 2295 }, 2296 }, 2297 [MLX5_QP_STATE_RTR] = { 2298 [MLX5_QP_STATE_RTS] = { 2299 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH | 2300 MLX5_QP_OPTPAR_RRE | 2301 MLX5_QP_OPTPAR_RAE | 2302 MLX5_QP_OPTPAR_RWE | 2303 MLX5_QP_OPTPAR_PM_STATE | 2304 MLX5_QP_OPTPAR_RNR_TIMEOUT, 2305 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH | 2306 MLX5_QP_OPTPAR_RWE | 2307 MLX5_QP_OPTPAR_PM_STATE, 2308 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY, 2309 }, 2310 }, 2311 [MLX5_QP_STATE_RTS] = { 2312 [MLX5_QP_STATE_RTS] = { 2313 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE | 2314 MLX5_QP_OPTPAR_RAE | 2315 MLX5_QP_OPTPAR_RWE | 2316 MLX5_QP_OPTPAR_RNR_TIMEOUT | 2317 MLX5_QP_OPTPAR_PM_STATE | 2318 MLX5_QP_OPTPAR_ALT_ADDR_PATH, 2319 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE | 2320 MLX5_QP_OPTPAR_PM_STATE | 2321 MLX5_QP_OPTPAR_ALT_ADDR_PATH, 2322 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY | 2323 MLX5_QP_OPTPAR_SRQN | 2324 MLX5_QP_OPTPAR_CQN_RCV, 2325 }, 2326 }, 2327 [MLX5_QP_STATE_SQER] = { 2328 [MLX5_QP_STATE_RTS] = { 2329 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY, 2330 [MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_Q_KEY, 2331 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE, 2332 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RNR_TIMEOUT | 2333 MLX5_QP_OPTPAR_RWE | 2334 MLX5_QP_OPTPAR_RAE | 2335 MLX5_QP_OPTPAR_RRE, 2336 }, 2337 }, 2338 }; 2339 2340 static int ib_nr_to_mlx5_nr(int ib_mask) 2341 { 2342 switch (ib_mask) { 2343 case IB_QP_STATE: 2344 return 0; 2345 case IB_QP_CUR_STATE: 2346 return 0; 2347 case IB_QP_EN_SQD_ASYNC_NOTIFY: 2348 return 0; 2349 case IB_QP_ACCESS_FLAGS: 2350 return MLX5_QP_OPTPAR_RWE | MLX5_QP_OPTPAR_RRE | 2351 MLX5_QP_OPTPAR_RAE; 2352 case IB_QP_PKEY_INDEX: 2353 return MLX5_QP_OPTPAR_PKEY_INDEX; 2354 case IB_QP_PORT: 2355 return MLX5_QP_OPTPAR_PRI_PORT; 2356 case IB_QP_QKEY: 2357 return MLX5_QP_OPTPAR_Q_KEY; 2358 case IB_QP_AV: 2359 return MLX5_QP_OPTPAR_PRIMARY_ADDR_PATH | 2360 MLX5_QP_OPTPAR_PRI_PORT; 2361 case IB_QP_PATH_MTU: 2362 return 0; 2363 case IB_QP_TIMEOUT: 2364 return MLX5_QP_OPTPAR_ACK_TIMEOUT; 2365 case IB_QP_RETRY_CNT: 2366 return MLX5_QP_OPTPAR_RETRY_COUNT; 2367 case IB_QP_RNR_RETRY: 2368 return MLX5_QP_OPTPAR_RNR_RETRY; 2369 case IB_QP_RQ_PSN: 2370 return 0; 2371 case IB_QP_MAX_QP_RD_ATOMIC: 2372 return MLX5_QP_OPTPAR_SRA_MAX; 2373 case IB_QP_ALT_PATH: 2374 return MLX5_QP_OPTPAR_ALT_ADDR_PATH; 2375 case IB_QP_MIN_RNR_TIMER: 2376 return MLX5_QP_OPTPAR_RNR_TIMEOUT; 2377 case IB_QP_SQ_PSN: 2378 return 0; 2379 case IB_QP_MAX_DEST_RD_ATOMIC: 2380 return MLX5_QP_OPTPAR_RRA_MAX | MLX5_QP_OPTPAR_RWE | 2381 MLX5_QP_OPTPAR_RRE | MLX5_QP_OPTPAR_RAE; 2382 case IB_QP_PATH_MIG_STATE: 2383 return MLX5_QP_OPTPAR_PM_STATE; 2384 case IB_QP_CAP: 2385 return 0; 2386 case IB_QP_DEST_QPN: 2387 return 0; 2388 } 2389 return 0; 2390 } 2391 2392 static int ib_mask_to_mlx5_opt(int ib_mask) 2393 { 2394 int result = 0; 2395 int i; 2396 2397 for (i = 0; i < 8 * sizeof(int); i++) { 2398 if ((1 << i) & ib_mask) 2399 result |= ib_nr_to_mlx5_nr(1 << i); 2400 } 2401 2402 return result; 2403 } 2404 2405 static int modify_raw_packet_qp_rq(struct mlx5_ib_dev *dev, 2406 struct mlx5_ib_rq *rq, int new_state, 2407 const struct mlx5_modify_raw_qp_param *raw_qp_param) 2408 { 2409 void *in; 2410 void *rqc; 2411 int inlen; 2412 int err; 2413 2414 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 2415 in = mlx5_vzalloc(inlen); 2416 if (!in) 2417 return -ENOMEM; 2418 2419 MLX5_SET(modify_rq_in, in, rq_state, rq->state); 2420 2421 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 2422 MLX5_SET(rqc, rqc, state, new_state); 2423 2424 if (raw_qp_param->set_mask & MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID) { 2425 if (MLX5_CAP_GEN(dev->mdev, modify_rq_counter_set_id)) { 2426 MLX5_SET64(modify_rq_in, in, modify_bitmask, 2427 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_MODIFY_RQ_COUNTER_SET_ID); 2428 MLX5_SET(rqc, rqc, counter_set_id, raw_qp_param->rq_q_ctr_id); 2429 } else 2430 pr_info_once("%s: RAW PACKET QP counters are not supported on current FW\n", 2431 dev->ib_dev.name); 2432 } 2433 2434 err = mlx5_core_modify_rq(dev->mdev, rq->base.mqp.qpn, in, inlen); 2435 if (err) 2436 goto out; 2437 2438 rq->state = new_state; 2439 2440 out: 2441 kvfree(in); 2442 return err; 2443 } 2444 2445 static int modify_raw_packet_qp_sq(struct mlx5_core_dev *dev, 2446 struct mlx5_ib_sq *sq, int new_state) 2447 { 2448 void *in; 2449 void *sqc; 2450 int inlen; 2451 int err; 2452 2453 inlen = MLX5_ST_SZ_BYTES(modify_sq_in); 2454 in = mlx5_vzalloc(inlen); 2455 if (!in) 2456 return -ENOMEM; 2457 2458 MLX5_SET(modify_sq_in, in, sq_state, sq->state); 2459 2460 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx); 2461 MLX5_SET(sqc, sqc, state, new_state); 2462 2463 err = mlx5_core_modify_sq(dev, sq->base.mqp.qpn, in, inlen); 2464 if (err) 2465 goto out; 2466 2467 sq->state = new_state; 2468 2469 out: 2470 kvfree(in); 2471 return err; 2472 } 2473 2474 static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, 2475 const struct mlx5_modify_raw_qp_param *raw_qp_param, 2476 u8 tx_affinity) 2477 { 2478 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp; 2479 struct mlx5_ib_rq *rq = &raw_packet_qp->rq; 2480 struct mlx5_ib_sq *sq = &raw_packet_qp->sq; 2481 int rq_state; 2482 int sq_state; 2483 int err; 2484 2485 switch (raw_qp_param->operation) { 2486 case MLX5_CMD_OP_RST2INIT_QP: 2487 rq_state = MLX5_RQC_STATE_RDY; 2488 sq_state = MLX5_SQC_STATE_RDY; 2489 break; 2490 case MLX5_CMD_OP_2ERR_QP: 2491 rq_state = MLX5_RQC_STATE_ERR; 2492 sq_state = MLX5_SQC_STATE_ERR; 2493 break; 2494 case MLX5_CMD_OP_2RST_QP: 2495 rq_state = MLX5_RQC_STATE_RST; 2496 sq_state = MLX5_SQC_STATE_RST; 2497 break; 2498 case MLX5_CMD_OP_INIT2INIT_QP: 2499 case MLX5_CMD_OP_INIT2RTR_QP: 2500 case MLX5_CMD_OP_RTR2RTS_QP: 2501 case MLX5_CMD_OP_RTS2RTS_QP: 2502 if (raw_qp_param->set_mask) 2503 return -EINVAL; 2504 else 2505 return 0; 2506 default: 2507 WARN_ON(1); 2508 return -EINVAL; 2509 } 2510 2511 if (qp->rq.wqe_cnt) { 2512 err = modify_raw_packet_qp_rq(dev, rq, rq_state, raw_qp_param); 2513 if (err) 2514 return err; 2515 } 2516 2517 if (qp->sq.wqe_cnt) { 2518 if (tx_affinity) { 2519 err = modify_raw_packet_tx_affinity(dev->mdev, sq, 2520 tx_affinity); 2521 if (err) 2522 return err; 2523 } 2524 2525 return modify_raw_packet_qp_sq(dev->mdev, sq, sq_state); 2526 } 2527 2528 return 0; 2529 } 2530 2531 static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, 2532 const struct ib_qp_attr *attr, int attr_mask, 2533 enum ib_qp_state cur_state, enum ib_qp_state new_state) 2534 { 2535 static const u16 optab[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE] = { 2536 [MLX5_QP_STATE_RST] = { 2537 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP, 2538 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP, 2539 [MLX5_QP_STATE_INIT] = MLX5_CMD_OP_RST2INIT_QP, 2540 }, 2541 [MLX5_QP_STATE_INIT] = { 2542 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP, 2543 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP, 2544 [MLX5_QP_STATE_INIT] = MLX5_CMD_OP_INIT2INIT_QP, 2545 [MLX5_QP_STATE_RTR] = MLX5_CMD_OP_INIT2RTR_QP, 2546 }, 2547 [MLX5_QP_STATE_RTR] = { 2548 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP, 2549 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP, 2550 [MLX5_QP_STATE_RTS] = MLX5_CMD_OP_RTR2RTS_QP, 2551 }, 2552 [MLX5_QP_STATE_RTS] = { 2553 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP, 2554 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP, 2555 [MLX5_QP_STATE_RTS] = MLX5_CMD_OP_RTS2RTS_QP, 2556 }, 2557 [MLX5_QP_STATE_SQD] = { 2558 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP, 2559 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP, 2560 }, 2561 [MLX5_QP_STATE_SQER] = { 2562 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP, 2563 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP, 2564 [MLX5_QP_STATE_RTS] = MLX5_CMD_OP_SQERR2RTS_QP, 2565 }, 2566 [MLX5_QP_STATE_ERR] = { 2567 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP, 2568 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP, 2569 } 2570 }; 2571 2572 struct mlx5_ib_dev *dev = to_mdev(ibqp->device); 2573 struct mlx5_ib_qp *qp = to_mqp(ibqp); 2574 struct mlx5_ib_qp_base *base = &qp->trans_qp.base; 2575 struct mlx5_ib_cq *send_cq, *recv_cq; 2576 struct mlx5_qp_context *context; 2577 struct mlx5_ib_pd *pd; 2578 struct mlx5_ib_port *mibport = NULL; 2579 enum mlx5_qp_state mlx5_cur, mlx5_new; 2580 enum mlx5_qp_optpar optpar; 2581 int sqd_event; 2582 int mlx5_st; 2583 int err; 2584 u16 op; 2585 u8 tx_affinity = 0; 2586 2587 context = kzalloc(sizeof(*context), GFP_KERNEL); 2588 if (!context) 2589 return -ENOMEM; 2590 2591 err = to_mlx5_st(ibqp->qp_type); 2592 if (err < 0) { 2593 mlx5_ib_dbg(dev, "unsupported qp type %d\n", ibqp->qp_type); 2594 goto out; 2595 } 2596 2597 context->flags = cpu_to_be32(err << 16); 2598 2599 if (!(attr_mask & IB_QP_PATH_MIG_STATE)) { 2600 context->flags |= cpu_to_be32(MLX5_QP_PM_MIGRATED << 11); 2601 } else { 2602 switch (attr->path_mig_state) { 2603 case IB_MIG_MIGRATED: 2604 context->flags |= cpu_to_be32(MLX5_QP_PM_MIGRATED << 11); 2605 break; 2606 case IB_MIG_REARM: 2607 context->flags |= cpu_to_be32(MLX5_QP_PM_REARM << 11); 2608 break; 2609 case IB_MIG_ARMED: 2610 context->flags |= cpu_to_be32(MLX5_QP_PM_ARMED << 11); 2611 break; 2612 } 2613 } 2614 2615 if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT)) { 2616 if ((ibqp->qp_type == IB_QPT_RC) || 2617 (ibqp->qp_type == IB_QPT_UD && 2618 !(qp->flags & MLX5_IB_QP_SQPN_QP1)) || 2619 (ibqp->qp_type == IB_QPT_UC) || 2620 (ibqp->qp_type == IB_QPT_RAW_PACKET) || 2621 (ibqp->qp_type == IB_QPT_XRC_INI) || 2622 (ibqp->qp_type == IB_QPT_XRC_TGT)) { 2623 if (mlx5_lag_is_active(dev->mdev)) { 2624 tx_affinity = (unsigned int)atomic_add_return(1, 2625 &dev->roce.next_port) % 2626 MLX5_MAX_PORTS + 1; 2627 context->flags |= cpu_to_be32(tx_affinity << 24); 2628 } 2629 } 2630 } 2631 2632 if (is_sqp(ibqp->qp_type)) { 2633 context->mtu_msgmax = (IB_MTU_256 << 5) | 8; 2634 } else if (ibqp->qp_type == IB_QPT_UD || 2635 ibqp->qp_type == MLX5_IB_QPT_REG_UMR) { 2636 context->mtu_msgmax = (IB_MTU_4096 << 5) | 12; 2637 } else if (attr_mask & IB_QP_PATH_MTU) { 2638 if (attr->path_mtu < IB_MTU_256 || 2639 attr->path_mtu > IB_MTU_4096) { 2640 mlx5_ib_warn(dev, "invalid mtu %d\n", attr->path_mtu); 2641 err = -EINVAL; 2642 goto out; 2643 } 2644 context->mtu_msgmax = (attr->path_mtu << 5) | 2645 (u8)MLX5_CAP_GEN(dev->mdev, log_max_msg); 2646 } 2647 2648 if (attr_mask & IB_QP_DEST_QPN) 2649 context->log_pg_sz_remote_qpn = cpu_to_be32(attr->dest_qp_num); 2650 2651 if (attr_mask & IB_QP_PKEY_INDEX) 2652 context->pri_path.pkey_index = cpu_to_be16(attr->pkey_index); 2653 2654 /* todo implement counter_index functionality */ 2655 2656 if (is_sqp(ibqp->qp_type)) 2657 context->pri_path.port = qp->port; 2658 2659 if (attr_mask & IB_QP_PORT) 2660 context->pri_path.port = attr->port_num; 2661 2662 if (attr_mask & IB_QP_AV) { 2663 err = mlx5_set_path(dev, qp, &attr->ah_attr, &context->pri_path, 2664 attr_mask & IB_QP_PORT ? attr->port_num : qp->port, 2665 attr_mask, 0, attr, false); 2666 if (err) 2667 goto out; 2668 } 2669 2670 if (attr_mask & IB_QP_TIMEOUT) 2671 context->pri_path.ackto_lt |= attr->timeout << 3; 2672 2673 if (attr_mask & IB_QP_ALT_PATH) { 2674 err = mlx5_set_path(dev, qp, &attr->alt_ah_attr, 2675 &context->alt_path, 2676 attr->alt_port_num, 2677 attr_mask | IB_QP_PKEY_INDEX | IB_QP_TIMEOUT, 2678 0, attr, true); 2679 if (err) 2680 goto out; 2681 } 2682 2683 pd = get_pd(qp); 2684 get_cqs(qp->ibqp.qp_type, qp->ibqp.send_cq, qp->ibqp.recv_cq, 2685 &send_cq, &recv_cq); 2686 2687 context->flags_pd = cpu_to_be32(pd ? pd->pdn : to_mpd(dev->devr.p0)->pdn); 2688 context->cqn_send = send_cq ? cpu_to_be32(send_cq->mcq.cqn) : 0; 2689 context->cqn_recv = recv_cq ? cpu_to_be32(recv_cq->mcq.cqn) : 0; 2690 context->params1 = cpu_to_be32(MLX5_IB_ACK_REQ_FREQ << 28); 2691 2692 if (attr_mask & IB_QP_RNR_RETRY) 2693 context->params1 |= cpu_to_be32(attr->rnr_retry << 13); 2694 2695 if (attr_mask & IB_QP_RETRY_CNT) 2696 context->params1 |= cpu_to_be32(attr->retry_cnt << 16); 2697 2698 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 2699 if (attr->max_rd_atomic) 2700 context->params1 |= 2701 cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21); 2702 } 2703 2704 if (attr_mask & IB_QP_SQ_PSN) 2705 context->next_send_psn = cpu_to_be32(attr->sq_psn); 2706 2707 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 2708 if (attr->max_dest_rd_atomic) 2709 context->params2 |= 2710 cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21); 2711 } 2712 2713 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) 2714 context->params2 |= to_mlx5_access_flags(qp, attr, attr_mask); 2715 2716 if (attr_mask & IB_QP_MIN_RNR_TIMER) 2717 context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24); 2718 2719 if (attr_mask & IB_QP_RQ_PSN) 2720 context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn); 2721 2722 if (attr_mask & IB_QP_QKEY) 2723 context->qkey = cpu_to_be32(attr->qkey); 2724 2725 if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) 2726 context->db_rec_addr = cpu_to_be64(qp->db.dma); 2727 2728 if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD && 2729 attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify) 2730 sqd_event = 1; 2731 else 2732 sqd_event = 0; 2733 2734 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { 2735 u8 port_num = (attr_mask & IB_QP_PORT ? attr->port_num : 2736 qp->port) - 1; 2737 mibport = &dev->port[port_num]; 2738 context->qp_counter_set_usr_page |= 2739 cpu_to_be32((u32)(mibport->q_cnt_id) << 24); 2740 } 2741 2742 if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) 2743 context->sq_crq_size |= cpu_to_be16(1 << 4); 2744 2745 if (qp->flags & MLX5_IB_QP_SQPN_QP1) 2746 context->deth_sqpn = cpu_to_be32(1); 2747 2748 mlx5_cur = to_mlx5_state(cur_state); 2749 mlx5_new = to_mlx5_state(new_state); 2750 mlx5_st = to_mlx5_st(ibqp->qp_type); 2751 if (mlx5_st < 0) 2752 goto out; 2753 2754 /* If moving to a reset or error state, we must disable page faults on 2755 * this QP and flush all current page faults. Otherwise a stale page 2756 * fault may attempt to work on this QP after it is reset and moved 2757 * again to RTS, and may cause the driver and the device to get out of 2758 * sync. */ 2759 if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR && 2760 (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR) && 2761 (qp->ibqp.qp_type != IB_QPT_RAW_PACKET)) 2762 mlx5_ib_qp_disable_pagefaults(qp); 2763 2764 if (mlx5_cur >= MLX5_QP_NUM_STATE || mlx5_new >= MLX5_QP_NUM_STATE || 2765 !optab[mlx5_cur][mlx5_new]) 2766 goto out; 2767 2768 op = optab[mlx5_cur][mlx5_new]; 2769 optpar = ib_mask_to_mlx5_opt(attr_mask); 2770 optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st]; 2771 2772 if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET) { 2773 struct mlx5_modify_raw_qp_param raw_qp_param = {}; 2774 2775 raw_qp_param.operation = op; 2776 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { 2777 raw_qp_param.rq_q_ctr_id = mibport->q_cnt_id; 2778 raw_qp_param.set_mask |= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID; 2779 } 2780 err = modify_raw_packet_qp(dev, qp, &raw_qp_param, tx_affinity); 2781 } else { 2782 err = mlx5_core_qp_modify(dev->mdev, op, optpar, context, 2783 &base->mqp); 2784 } 2785 2786 if (err) 2787 goto out; 2788 2789 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT && 2790 (qp->ibqp.qp_type != IB_QPT_RAW_PACKET)) 2791 mlx5_ib_qp_enable_pagefaults(qp); 2792 2793 qp->state = new_state; 2794 2795 if (attr_mask & IB_QP_ACCESS_FLAGS) 2796 qp->trans_qp.atomic_rd_en = attr->qp_access_flags; 2797 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) 2798 qp->trans_qp.resp_depth = attr->max_dest_rd_atomic; 2799 if (attr_mask & IB_QP_PORT) 2800 qp->port = attr->port_num; 2801 if (attr_mask & IB_QP_ALT_PATH) 2802 qp->trans_qp.alt_port = attr->alt_port_num; 2803 2804 /* 2805 * If we moved a kernel QP to RESET, clean up all old CQ 2806 * entries and reinitialize the QP. 2807 */ 2808 if (new_state == IB_QPS_RESET && !ibqp->uobject) { 2809 mlx5_ib_cq_clean(recv_cq, base->mqp.qpn, 2810 ibqp->srq ? to_msrq(ibqp->srq) : NULL); 2811 if (send_cq != recv_cq) 2812 mlx5_ib_cq_clean(send_cq, base->mqp.qpn, NULL); 2813 2814 qp->rq.head = 0; 2815 qp->rq.tail = 0; 2816 qp->sq.head = 0; 2817 qp->sq.tail = 0; 2818 qp->sq.cur_post = 0; 2819 qp->sq.last_poll = 0; 2820 qp->db.db[MLX5_RCV_DBR] = 0; 2821 qp->db.db[MLX5_SND_DBR] = 0; 2822 } 2823 2824 out: 2825 kfree(context); 2826 return err; 2827 } 2828 2829 int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 2830 int attr_mask, struct ib_udata *udata) 2831 { 2832 struct mlx5_ib_dev *dev = to_mdev(ibqp->device); 2833 struct mlx5_ib_qp *qp = to_mqp(ibqp); 2834 enum ib_qp_type qp_type; 2835 enum ib_qp_state cur_state, new_state; 2836 int err = -EINVAL; 2837 int port; 2838 enum rdma_link_layer ll = IB_LINK_LAYER_UNSPECIFIED; 2839 2840 if (ibqp->rwq_ind_tbl) 2841 return -ENOSYS; 2842 2843 if (unlikely(ibqp->qp_type == IB_QPT_GSI)) 2844 return mlx5_ib_gsi_modify_qp(ibqp, attr, attr_mask); 2845 2846 qp_type = (unlikely(ibqp->qp_type == MLX5_IB_QPT_HW_GSI)) ? 2847 IB_QPT_GSI : ibqp->qp_type; 2848 2849 mutex_lock(&qp->mutex); 2850 2851 cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state; 2852 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; 2853 2854 if (!(cur_state == new_state && cur_state == IB_QPS_RESET)) { 2855 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port; 2856 ll = dev->ib_dev.get_link_layer(&dev->ib_dev, port); 2857 } 2858 2859 if (qp_type != MLX5_IB_QPT_REG_UMR && 2860 !ib_modify_qp_is_ok(cur_state, new_state, qp_type, attr_mask, ll)) { 2861 mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n", 2862 cur_state, new_state, ibqp->qp_type, attr_mask); 2863 goto out; 2864 } 2865 2866 if ((attr_mask & IB_QP_PORT) && 2867 (attr->port_num == 0 || 2868 attr->port_num > MLX5_CAP_GEN(dev->mdev, num_ports))) { 2869 mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n", 2870 attr->port_num, dev->num_ports); 2871 goto out; 2872 } 2873 2874 if (attr_mask & IB_QP_PKEY_INDEX) { 2875 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port; 2876 if (attr->pkey_index >= 2877 dev->mdev->port_caps[port - 1].pkey_table_len) { 2878 mlx5_ib_dbg(dev, "invalid pkey index %d\n", 2879 attr->pkey_index); 2880 goto out; 2881 } 2882 } 2883 2884 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && 2885 attr->max_rd_atomic > 2886 (1 << MLX5_CAP_GEN(dev->mdev, log_max_ra_res_qp))) { 2887 mlx5_ib_dbg(dev, "invalid max_rd_atomic value %d\n", 2888 attr->max_rd_atomic); 2889 goto out; 2890 } 2891 2892 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && 2893 attr->max_dest_rd_atomic > 2894 (1 << MLX5_CAP_GEN(dev->mdev, log_max_ra_req_qp))) { 2895 mlx5_ib_dbg(dev, "invalid max_dest_rd_atomic value %d\n", 2896 attr->max_dest_rd_atomic); 2897 goto out; 2898 } 2899 2900 if (cur_state == new_state && cur_state == IB_QPS_RESET) { 2901 err = 0; 2902 goto out; 2903 } 2904 2905 err = __mlx5_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state); 2906 2907 out: 2908 mutex_unlock(&qp->mutex); 2909 return err; 2910 } 2911 2912 static int mlx5_wq_overflow(struct mlx5_ib_wq *wq, int nreq, struct ib_cq *ib_cq) 2913 { 2914 struct mlx5_ib_cq *cq; 2915 unsigned cur; 2916 2917 cur = wq->head - wq->tail; 2918 if (likely(cur + nreq < wq->max_post)) 2919 return 0; 2920 2921 cq = to_mcq(ib_cq); 2922 spin_lock(&cq->lock); 2923 cur = wq->head - wq->tail; 2924 spin_unlock(&cq->lock); 2925 2926 return cur + nreq >= wq->max_post; 2927 } 2928 2929 static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg, 2930 u64 remote_addr, u32 rkey) 2931 { 2932 rseg->raddr = cpu_to_be64(remote_addr); 2933 rseg->rkey = cpu_to_be32(rkey); 2934 rseg->reserved = 0; 2935 } 2936 2937 static void *set_eth_seg(struct mlx5_wqe_eth_seg *eseg, 2938 struct ib_send_wr *wr, void *qend, 2939 struct mlx5_ib_qp *qp, int *size) 2940 { 2941 void *seg = eseg; 2942 2943 memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg)); 2944 2945 if (wr->send_flags & IB_SEND_IP_CSUM) 2946 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM | 2947 MLX5_ETH_WQE_L4_CSUM; 2948 2949 seg += sizeof(struct mlx5_wqe_eth_seg); 2950 *size += sizeof(struct mlx5_wqe_eth_seg) / 16; 2951 2952 if (wr->opcode == IB_WR_LSO) { 2953 struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr); 2954 int size_of_inl_hdr_start = sizeof(eseg->inline_hdr_start); 2955 u64 left, leftlen, copysz; 2956 void *pdata = ud_wr->header; 2957 2958 left = ud_wr->hlen; 2959 eseg->mss = cpu_to_be16(ud_wr->mss); 2960 eseg->inline_hdr_sz = cpu_to_be16(left); 2961 2962 /* 2963 * check if there is space till the end of queue, if yes, 2964 * copy all in one shot, otherwise copy till the end of queue, 2965 * rollback and than the copy the left 2966 */ 2967 leftlen = qend - (void *)eseg->inline_hdr_start; 2968 copysz = min_t(u64, leftlen, left); 2969 2970 memcpy(seg - size_of_inl_hdr_start, pdata, copysz); 2971 2972 if (likely(copysz > size_of_inl_hdr_start)) { 2973 seg += ALIGN(copysz - size_of_inl_hdr_start, 16); 2974 *size += ALIGN(copysz - size_of_inl_hdr_start, 16) / 16; 2975 } 2976 2977 if (unlikely(copysz < left)) { /* the last wqe in the queue */ 2978 seg = mlx5_get_send_wqe(qp, 0); 2979 left -= copysz; 2980 pdata += copysz; 2981 memcpy(seg, pdata, left); 2982 seg += ALIGN(left, 16); 2983 *size += ALIGN(left, 16) / 16; 2984 } 2985 } 2986 2987 return seg; 2988 } 2989 2990 static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg, 2991 struct ib_send_wr *wr) 2992 { 2993 memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av)); 2994 dseg->av.dqp_dct = cpu_to_be32(ud_wr(wr)->remote_qpn | MLX5_EXTENDED_UD_AV); 2995 dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey); 2996 } 2997 2998 static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg) 2999 { 3000 dseg->byte_count = cpu_to_be32(sg->length); 3001 dseg->lkey = cpu_to_be32(sg->lkey); 3002 dseg->addr = cpu_to_be64(sg->addr); 3003 } 3004 3005 static __be16 get_klm_octo(int npages) 3006 { 3007 return cpu_to_be16(ALIGN(npages, 8) / 2); 3008 } 3009 3010 static __be64 frwr_mkey_mask(void) 3011 { 3012 u64 result; 3013 3014 result = MLX5_MKEY_MASK_LEN | 3015 MLX5_MKEY_MASK_PAGE_SIZE | 3016 MLX5_MKEY_MASK_START_ADDR | 3017 MLX5_MKEY_MASK_EN_RINVAL | 3018 MLX5_MKEY_MASK_KEY | 3019 MLX5_MKEY_MASK_LR | 3020 MLX5_MKEY_MASK_LW | 3021 MLX5_MKEY_MASK_RR | 3022 MLX5_MKEY_MASK_RW | 3023 MLX5_MKEY_MASK_A | 3024 MLX5_MKEY_MASK_SMALL_FENCE | 3025 MLX5_MKEY_MASK_FREE; 3026 3027 return cpu_to_be64(result); 3028 } 3029 3030 static __be64 sig_mkey_mask(void) 3031 { 3032 u64 result; 3033 3034 result = MLX5_MKEY_MASK_LEN | 3035 MLX5_MKEY_MASK_PAGE_SIZE | 3036 MLX5_MKEY_MASK_START_ADDR | 3037 MLX5_MKEY_MASK_EN_SIGERR | 3038 MLX5_MKEY_MASK_EN_RINVAL | 3039 MLX5_MKEY_MASK_KEY | 3040 MLX5_MKEY_MASK_LR | 3041 MLX5_MKEY_MASK_LW | 3042 MLX5_MKEY_MASK_RR | 3043 MLX5_MKEY_MASK_RW | 3044 MLX5_MKEY_MASK_SMALL_FENCE | 3045 MLX5_MKEY_MASK_FREE | 3046 MLX5_MKEY_MASK_BSF_EN; 3047 3048 return cpu_to_be64(result); 3049 } 3050 3051 static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr, 3052 struct mlx5_ib_mr *mr) 3053 { 3054 int ndescs = mr->ndescs; 3055 3056 memset(umr, 0, sizeof(*umr)); 3057 3058 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS) 3059 /* KLMs take twice the size of MTTs */ 3060 ndescs *= 2; 3061 3062 umr->flags = MLX5_UMR_CHECK_NOT_FREE; 3063 umr->klm_octowords = get_klm_octo(ndescs); 3064 umr->mkey_mask = frwr_mkey_mask(); 3065 } 3066 3067 static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr) 3068 { 3069 memset(umr, 0, sizeof(*umr)); 3070 umr->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE); 3071 umr->flags = 1 << 7; 3072 } 3073 3074 static __be64 get_umr_reg_mr_mask(void) 3075 { 3076 u64 result; 3077 3078 result = MLX5_MKEY_MASK_LEN | 3079 MLX5_MKEY_MASK_PAGE_SIZE | 3080 MLX5_MKEY_MASK_START_ADDR | 3081 MLX5_MKEY_MASK_PD | 3082 MLX5_MKEY_MASK_LR | 3083 MLX5_MKEY_MASK_LW | 3084 MLX5_MKEY_MASK_KEY | 3085 MLX5_MKEY_MASK_RR | 3086 MLX5_MKEY_MASK_RW | 3087 MLX5_MKEY_MASK_A | 3088 MLX5_MKEY_MASK_FREE; 3089 3090 return cpu_to_be64(result); 3091 } 3092 3093 static __be64 get_umr_unreg_mr_mask(void) 3094 { 3095 u64 result; 3096 3097 result = MLX5_MKEY_MASK_FREE; 3098 3099 return cpu_to_be64(result); 3100 } 3101 3102 static __be64 get_umr_update_mtt_mask(void) 3103 { 3104 u64 result; 3105 3106 result = MLX5_MKEY_MASK_FREE; 3107 3108 return cpu_to_be64(result); 3109 } 3110 3111 static __be64 get_umr_update_translation_mask(void) 3112 { 3113 u64 result; 3114 3115 result = MLX5_MKEY_MASK_LEN | 3116 MLX5_MKEY_MASK_PAGE_SIZE | 3117 MLX5_MKEY_MASK_START_ADDR | 3118 MLX5_MKEY_MASK_KEY | 3119 MLX5_MKEY_MASK_FREE; 3120 3121 return cpu_to_be64(result); 3122 } 3123 3124 static __be64 get_umr_update_access_mask(void) 3125 { 3126 u64 result; 3127 3128 result = MLX5_MKEY_MASK_LW | 3129 MLX5_MKEY_MASK_RR | 3130 MLX5_MKEY_MASK_RW | 3131 MLX5_MKEY_MASK_A | 3132 MLX5_MKEY_MASK_KEY | 3133 MLX5_MKEY_MASK_FREE; 3134 3135 return cpu_to_be64(result); 3136 } 3137 3138 static __be64 get_umr_update_pd_mask(void) 3139 { 3140 u64 result; 3141 3142 result = MLX5_MKEY_MASK_PD | 3143 MLX5_MKEY_MASK_KEY | 3144 MLX5_MKEY_MASK_FREE; 3145 3146 return cpu_to_be64(result); 3147 } 3148 3149 static void set_reg_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr, 3150 struct ib_send_wr *wr) 3151 { 3152 struct mlx5_umr_wr *umrwr = umr_wr(wr); 3153 3154 memset(umr, 0, sizeof(*umr)); 3155 3156 if (wr->send_flags & MLX5_IB_SEND_UMR_FAIL_IF_FREE) 3157 umr->flags = MLX5_UMR_CHECK_FREE; /* fail if free */ 3158 else 3159 umr->flags = MLX5_UMR_CHECK_NOT_FREE; /* fail if not free */ 3160 3161 if (!(wr->send_flags & MLX5_IB_SEND_UMR_UNREG)) { 3162 umr->klm_octowords = get_klm_octo(umrwr->npages); 3163 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_MTT) { 3164 umr->mkey_mask = get_umr_update_mtt_mask(); 3165 umr->bsf_octowords = get_klm_octo(umrwr->target.offset); 3166 umr->flags |= MLX5_UMR_TRANSLATION_OFFSET_EN; 3167 } 3168 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION) 3169 umr->mkey_mask |= get_umr_update_translation_mask(); 3170 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_ACCESS) 3171 umr->mkey_mask |= get_umr_update_access_mask(); 3172 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD) 3173 umr->mkey_mask |= get_umr_update_pd_mask(); 3174 if (!umr->mkey_mask) 3175 umr->mkey_mask = get_umr_reg_mr_mask(); 3176 } else { 3177 umr->mkey_mask = get_umr_unreg_mr_mask(); 3178 } 3179 3180 if (!wr->num_sge) 3181 umr->flags |= MLX5_UMR_INLINE; 3182 } 3183 3184 static u8 get_umr_flags(int acc) 3185 { 3186 return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC : 0) | 3187 (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) | 3188 (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) | 3189 (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) | 3190 MLX5_PERM_LOCAL_READ | MLX5_PERM_UMR_EN; 3191 } 3192 3193 static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg, 3194 struct mlx5_ib_mr *mr, 3195 u32 key, int access) 3196 { 3197 int ndescs = ALIGN(mr->ndescs, 8) >> 1; 3198 3199 memset(seg, 0, sizeof(*seg)); 3200 3201 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_MTT) 3202 seg->log2_page_size = ilog2(mr->ibmr.page_size); 3203 else if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS) 3204 /* KLMs take twice the size of MTTs */ 3205 ndescs *= 2; 3206 3207 seg->flags = get_umr_flags(access) | mr->access_mode; 3208 seg->qpn_mkey7_0 = cpu_to_be32((key & 0xff) | 0xffffff00); 3209 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL); 3210 seg->start_addr = cpu_to_be64(mr->ibmr.iova); 3211 seg->len = cpu_to_be64(mr->ibmr.length); 3212 seg->xlt_oct_size = cpu_to_be32(ndescs); 3213 } 3214 3215 static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg) 3216 { 3217 memset(seg, 0, sizeof(*seg)); 3218 seg->status = MLX5_MKEY_STATUS_FREE; 3219 } 3220 3221 static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg, struct ib_send_wr *wr) 3222 { 3223 struct mlx5_umr_wr *umrwr = umr_wr(wr); 3224 3225 memset(seg, 0, sizeof(*seg)); 3226 if (wr->send_flags & MLX5_IB_SEND_UMR_UNREG) { 3227 seg->status = MLX5_MKEY_STATUS_FREE; 3228 return; 3229 } 3230 3231 seg->flags = convert_access(umrwr->access_flags); 3232 if (!(wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_MTT)) { 3233 if (umrwr->pd) 3234 seg->flags_pd = cpu_to_be32(to_mpd(umrwr->pd)->pdn); 3235 seg->start_addr = cpu_to_be64(umrwr->target.virt_addr); 3236 } 3237 seg->len = cpu_to_be64(umrwr->length); 3238 seg->log2_page_size = umrwr->page_shift; 3239 seg->qpn_mkey7_0 = cpu_to_be32(0xffffff00 | 3240 mlx5_mkey_variant(umrwr->mkey)); 3241 } 3242 3243 static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg, 3244 struct mlx5_ib_mr *mr, 3245 struct mlx5_ib_pd *pd) 3246 { 3247 int bcount = mr->desc_size * mr->ndescs; 3248 3249 dseg->addr = cpu_to_be64(mr->desc_map); 3250 dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64)); 3251 dseg->lkey = cpu_to_be32(pd->ibpd.local_dma_lkey); 3252 } 3253 3254 static __be32 send_ieth(struct ib_send_wr *wr) 3255 { 3256 switch (wr->opcode) { 3257 case IB_WR_SEND_WITH_IMM: 3258 case IB_WR_RDMA_WRITE_WITH_IMM: 3259 return wr->ex.imm_data; 3260 3261 case IB_WR_SEND_WITH_INV: 3262 return cpu_to_be32(wr->ex.invalidate_rkey); 3263 3264 default: 3265 return 0; 3266 } 3267 } 3268 3269 static u8 calc_sig(void *wqe, int size) 3270 { 3271 u8 *p = wqe; 3272 u8 res = 0; 3273 int i; 3274 3275 for (i = 0; i < size; i++) 3276 res ^= p[i]; 3277 3278 return ~res; 3279 } 3280 3281 static u8 wq_sig(void *wqe) 3282 { 3283 return calc_sig(wqe, (*((u8 *)wqe + 8) & 0x3f) << 4); 3284 } 3285 3286 static int set_data_inl_seg(struct mlx5_ib_qp *qp, struct ib_send_wr *wr, 3287 void *wqe, int *sz) 3288 { 3289 struct mlx5_wqe_inline_seg *seg; 3290 void *qend = qp->sq.qend; 3291 void *addr; 3292 int inl = 0; 3293 int copy; 3294 int len; 3295 int i; 3296 3297 seg = wqe; 3298 wqe += sizeof(*seg); 3299 for (i = 0; i < wr->num_sge; i++) { 3300 addr = (void *)(unsigned long)(wr->sg_list[i].addr); 3301 len = wr->sg_list[i].length; 3302 inl += len; 3303 3304 if (unlikely(inl > qp->max_inline_data)) 3305 return -ENOMEM; 3306 3307 if (unlikely(wqe + len > qend)) { 3308 copy = qend - wqe; 3309 memcpy(wqe, addr, copy); 3310 addr += copy; 3311 len -= copy; 3312 wqe = mlx5_get_send_wqe(qp, 0); 3313 } 3314 memcpy(wqe, addr, len); 3315 wqe += len; 3316 } 3317 3318 seg->byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG); 3319 3320 *sz = ALIGN(inl + sizeof(seg->byte_count), 16) / 16; 3321 3322 return 0; 3323 } 3324 3325 static u16 prot_field_size(enum ib_signature_type type) 3326 { 3327 switch (type) { 3328 case IB_SIG_TYPE_T10_DIF: 3329 return MLX5_DIF_SIZE; 3330 default: 3331 return 0; 3332 } 3333 } 3334 3335 static u8 bs_selector(int block_size) 3336 { 3337 switch (block_size) { 3338 case 512: return 0x1; 3339 case 520: return 0x2; 3340 case 4096: return 0x3; 3341 case 4160: return 0x4; 3342 case 1073741824: return 0x5; 3343 default: return 0; 3344 } 3345 } 3346 3347 static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain, 3348 struct mlx5_bsf_inl *inl) 3349 { 3350 /* Valid inline section and allow BSF refresh */ 3351 inl->vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID | 3352 MLX5_BSF_REFRESH_DIF); 3353 inl->dif_apptag = cpu_to_be16(domain->sig.dif.app_tag); 3354 inl->dif_reftag = cpu_to_be32(domain->sig.dif.ref_tag); 3355 /* repeating block */ 3356 inl->rp_inv_seed = MLX5_BSF_REPEAT_BLOCK; 3357 inl->sig_type = domain->sig.dif.bg_type == IB_T10DIF_CRC ? 3358 MLX5_DIF_CRC : MLX5_DIF_IPCS; 3359 3360 if (domain->sig.dif.ref_remap) 3361 inl->dif_inc_ref_guard_check |= MLX5_BSF_INC_REFTAG; 3362 3363 if (domain->sig.dif.app_escape) { 3364 if (domain->sig.dif.ref_escape) 3365 inl->dif_inc_ref_guard_check |= MLX5_BSF_APPREF_ESCAPE; 3366 else 3367 inl->dif_inc_ref_guard_check |= MLX5_BSF_APPTAG_ESCAPE; 3368 } 3369 3370 inl->dif_app_bitmask_check = 3371 cpu_to_be16(domain->sig.dif.apptag_check_mask); 3372 } 3373 3374 static int mlx5_set_bsf(struct ib_mr *sig_mr, 3375 struct ib_sig_attrs *sig_attrs, 3376 struct mlx5_bsf *bsf, u32 data_size) 3377 { 3378 struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)->sig; 3379 struct mlx5_bsf_basic *basic = &bsf->basic; 3380 struct ib_sig_domain *mem = &sig_attrs->mem; 3381 struct ib_sig_domain *wire = &sig_attrs->wire; 3382 3383 memset(bsf, 0, sizeof(*bsf)); 3384 3385 /* Basic + Extended + Inline */ 3386 basic->bsf_size_sbs = 1 << 7; 3387 /* Input domain check byte mask */ 3388 basic->check_byte_mask = sig_attrs->check_mask; 3389 basic->raw_data_size = cpu_to_be32(data_size); 3390 3391 /* Memory domain */ 3392 switch (sig_attrs->mem.sig_type) { 3393 case IB_SIG_TYPE_NONE: 3394 break; 3395 case IB_SIG_TYPE_T10_DIF: 3396 basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval); 3397 basic->m_bfs_psv = cpu_to_be32(msig->psv_memory.psv_idx); 3398 mlx5_fill_inl_bsf(mem, &bsf->m_inl); 3399 break; 3400 default: 3401 return -EINVAL; 3402 } 3403 3404 /* Wire domain */ 3405 switch (sig_attrs->wire.sig_type) { 3406 case IB_SIG_TYPE_NONE: 3407 break; 3408 case IB_SIG_TYPE_T10_DIF: 3409 if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval && 3410 mem->sig_type == wire->sig_type) { 3411 /* Same block structure */ 3412 basic->bsf_size_sbs |= 1 << 4; 3413 if (mem->sig.dif.bg_type == wire->sig.dif.bg_type) 3414 basic->wire.copy_byte_mask |= MLX5_CPY_GRD_MASK; 3415 if (mem->sig.dif.app_tag == wire->sig.dif.app_tag) 3416 basic->wire.copy_byte_mask |= MLX5_CPY_APP_MASK; 3417 if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag) 3418 basic->wire.copy_byte_mask |= MLX5_CPY_REF_MASK; 3419 } else 3420 basic->wire.bs_selector = bs_selector(wire->sig.dif.pi_interval); 3421 3422 basic->w_bfs_psv = cpu_to_be32(msig->psv_wire.psv_idx); 3423 mlx5_fill_inl_bsf(wire, &bsf->w_inl); 3424 break; 3425 default: 3426 return -EINVAL; 3427 } 3428 3429 return 0; 3430 } 3431 3432 static int set_sig_data_segment(struct ib_sig_handover_wr *wr, 3433 struct mlx5_ib_qp *qp, void **seg, int *size) 3434 { 3435 struct ib_sig_attrs *sig_attrs = wr->sig_attrs; 3436 struct ib_mr *sig_mr = wr->sig_mr; 3437 struct mlx5_bsf *bsf; 3438 u32 data_len = wr->wr.sg_list->length; 3439 u32 data_key = wr->wr.sg_list->lkey; 3440 u64 data_va = wr->wr.sg_list->addr; 3441 int ret; 3442 int wqe_size; 3443 3444 if (!wr->prot || 3445 (data_key == wr->prot->lkey && 3446 data_va == wr->prot->addr && 3447 data_len == wr->prot->length)) { 3448 /** 3449 * Source domain doesn't contain signature information 3450 * or data and protection are interleaved in memory. 3451 * So need construct: 3452 * ------------------ 3453 * | data_klm | 3454 * ------------------ 3455 * | BSF | 3456 * ------------------ 3457 **/ 3458 struct mlx5_klm *data_klm = *seg; 3459 3460 data_klm->bcount = cpu_to_be32(data_len); 3461 data_klm->key = cpu_to_be32(data_key); 3462 data_klm->va = cpu_to_be64(data_va); 3463 wqe_size = ALIGN(sizeof(*data_klm), 64); 3464 } else { 3465 /** 3466 * Source domain contains signature information 3467 * So need construct a strided block format: 3468 * --------------------------- 3469 * | stride_block_ctrl | 3470 * --------------------------- 3471 * | data_klm | 3472 * --------------------------- 3473 * | prot_klm | 3474 * --------------------------- 3475 * | BSF | 3476 * --------------------------- 3477 **/ 3478 struct mlx5_stride_block_ctrl_seg *sblock_ctrl; 3479 struct mlx5_stride_block_entry *data_sentry; 3480 struct mlx5_stride_block_entry *prot_sentry; 3481 u32 prot_key = wr->prot->lkey; 3482 u64 prot_va = wr->prot->addr; 3483 u16 block_size = sig_attrs->mem.sig.dif.pi_interval; 3484 int prot_size; 3485 3486 sblock_ctrl = *seg; 3487 data_sentry = (void *)sblock_ctrl + sizeof(*sblock_ctrl); 3488 prot_sentry = (void *)data_sentry + sizeof(*data_sentry); 3489 3490 prot_size = prot_field_size(sig_attrs->mem.sig_type); 3491 if (!prot_size) { 3492 pr_err("Bad block size given: %u\n", block_size); 3493 return -EINVAL; 3494 } 3495 sblock_ctrl->bcount_per_cycle = cpu_to_be32(block_size + 3496 prot_size); 3497 sblock_ctrl->op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP); 3498 sblock_ctrl->repeat_count = cpu_to_be32(data_len / block_size); 3499 sblock_ctrl->num_entries = cpu_to_be16(2); 3500 3501 data_sentry->bcount = cpu_to_be16(block_size); 3502 data_sentry->key = cpu_to_be32(data_key); 3503 data_sentry->va = cpu_to_be64(data_va); 3504 data_sentry->stride = cpu_to_be16(block_size); 3505 3506 prot_sentry->bcount = cpu_to_be16(prot_size); 3507 prot_sentry->key = cpu_to_be32(prot_key); 3508 prot_sentry->va = cpu_to_be64(prot_va); 3509 prot_sentry->stride = cpu_to_be16(prot_size); 3510 3511 wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) + 3512 sizeof(*prot_sentry), 64); 3513 } 3514 3515 *seg += wqe_size; 3516 *size += wqe_size / 16; 3517 if (unlikely((*seg == qp->sq.qend))) 3518 *seg = mlx5_get_send_wqe(qp, 0); 3519 3520 bsf = *seg; 3521 ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len); 3522 if (ret) 3523 return -EINVAL; 3524 3525 *seg += sizeof(*bsf); 3526 *size += sizeof(*bsf) / 16; 3527 if (unlikely((*seg == qp->sq.qend))) 3528 *seg = mlx5_get_send_wqe(qp, 0); 3529 3530 return 0; 3531 } 3532 3533 static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg, 3534 struct ib_sig_handover_wr *wr, u32 nelements, 3535 u32 length, u32 pdn) 3536 { 3537 struct ib_mr *sig_mr = wr->sig_mr; 3538 u32 sig_key = sig_mr->rkey; 3539 u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1; 3540 3541 memset(seg, 0, sizeof(*seg)); 3542 3543 seg->flags = get_umr_flags(wr->access_flags) | 3544 MLX5_MKC_ACCESS_MODE_KLMS; 3545 seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00); 3546 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 | 3547 MLX5_MKEY_BSF_EN | pdn); 3548 seg->len = cpu_to_be64(length); 3549 seg->xlt_oct_size = cpu_to_be32(be16_to_cpu(get_klm_octo(nelements))); 3550 seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE); 3551 } 3552 3553 static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr, 3554 u32 nelements) 3555 { 3556 memset(umr, 0, sizeof(*umr)); 3557 3558 umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE; 3559 umr->klm_octowords = get_klm_octo(nelements); 3560 umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE); 3561 umr->mkey_mask = sig_mkey_mask(); 3562 } 3563 3564 3565 static int set_sig_umr_wr(struct ib_send_wr *send_wr, struct mlx5_ib_qp *qp, 3566 void **seg, int *size) 3567 { 3568 struct ib_sig_handover_wr *wr = sig_handover_wr(send_wr); 3569 struct mlx5_ib_mr *sig_mr = to_mmr(wr->sig_mr); 3570 u32 pdn = get_pd(qp)->pdn; 3571 u32 klm_oct_size; 3572 int region_len, ret; 3573 3574 if (unlikely(wr->wr.num_sge != 1) || 3575 unlikely(wr->access_flags & IB_ACCESS_REMOTE_ATOMIC) || 3576 unlikely(!sig_mr->sig) || unlikely(!qp->signature_en) || 3577 unlikely(!sig_mr->sig->sig_status_checked)) 3578 return -EINVAL; 3579 3580 /* length of the protected region, data + protection */ 3581 region_len = wr->wr.sg_list->length; 3582 if (wr->prot && 3583 (wr->prot->lkey != wr->wr.sg_list->lkey || 3584 wr->prot->addr != wr->wr.sg_list->addr || 3585 wr->prot->length != wr->wr.sg_list->length)) 3586 region_len += wr->prot->length; 3587 3588 /** 3589 * KLM octoword size - if protection was provided 3590 * then we use strided block format (3 octowords), 3591 * else we use single KLM (1 octoword) 3592 **/ 3593 klm_oct_size = wr->prot ? 3 : 1; 3594 3595 set_sig_umr_segment(*seg, klm_oct_size); 3596 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg); 3597 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; 3598 if (unlikely((*seg == qp->sq.qend))) 3599 *seg = mlx5_get_send_wqe(qp, 0); 3600 3601 set_sig_mkey_segment(*seg, wr, klm_oct_size, region_len, pdn); 3602 *seg += sizeof(struct mlx5_mkey_seg); 3603 *size += sizeof(struct mlx5_mkey_seg) / 16; 3604 if (unlikely((*seg == qp->sq.qend))) 3605 *seg = mlx5_get_send_wqe(qp, 0); 3606 3607 ret = set_sig_data_segment(wr, qp, seg, size); 3608 if (ret) 3609 return ret; 3610 3611 sig_mr->sig->sig_status_checked = false; 3612 return 0; 3613 } 3614 3615 static int set_psv_wr(struct ib_sig_domain *domain, 3616 u32 psv_idx, void **seg, int *size) 3617 { 3618 struct mlx5_seg_set_psv *psv_seg = *seg; 3619 3620 memset(psv_seg, 0, sizeof(*psv_seg)); 3621 psv_seg->psv_num = cpu_to_be32(psv_idx); 3622 switch (domain->sig_type) { 3623 case IB_SIG_TYPE_NONE: 3624 break; 3625 case IB_SIG_TYPE_T10_DIF: 3626 psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 | 3627 domain->sig.dif.app_tag); 3628 psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag); 3629 break; 3630 default: 3631 pr_err("Bad signature type given.\n"); 3632 return 1; 3633 } 3634 3635 *seg += sizeof(*psv_seg); 3636 *size += sizeof(*psv_seg) / 16; 3637 3638 return 0; 3639 } 3640 3641 static int set_reg_wr(struct mlx5_ib_qp *qp, 3642 struct ib_reg_wr *wr, 3643 void **seg, int *size) 3644 { 3645 struct mlx5_ib_mr *mr = to_mmr(wr->mr); 3646 struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd); 3647 3648 if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) { 3649 mlx5_ib_warn(to_mdev(qp->ibqp.device), 3650 "Invalid IB_SEND_INLINE send flag\n"); 3651 return -EINVAL; 3652 } 3653 3654 set_reg_umr_seg(*seg, mr); 3655 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg); 3656 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; 3657 if (unlikely((*seg == qp->sq.qend))) 3658 *seg = mlx5_get_send_wqe(qp, 0); 3659 3660 set_reg_mkey_seg(*seg, mr, wr->key, wr->access); 3661 *seg += sizeof(struct mlx5_mkey_seg); 3662 *size += sizeof(struct mlx5_mkey_seg) / 16; 3663 if (unlikely((*seg == qp->sq.qend))) 3664 *seg = mlx5_get_send_wqe(qp, 0); 3665 3666 set_reg_data_seg(*seg, mr, pd); 3667 *seg += sizeof(struct mlx5_wqe_data_seg); 3668 *size += (sizeof(struct mlx5_wqe_data_seg) / 16); 3669 3670 return 0; 3671 } 3672 3673 static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size) 3674 { 3675 set_linv_umr_seg(*seg); 3676 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg); 3677 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; 3678 if (unlikely((*seg == qp->sq.qend))) 3679 *seg = mlx5_get_send_wqe(qp, 0); 3680 set_linv_mkey_seg(*seg); 3681 *seg += sizeof(struct mlx5_mkey_seg); 3682 *size += sizeof(struct mlx5_mkey_seg) / 16; 3683 if (unlikely((*seg == qp->sq.qend))) 3684 *seg = mlx5_get_send_wqe(qp, 0); 3685 } 3686 3687 static void dump_wqe(struct mlx5_ib_qp *qp, int idx, int size_16) 3688 { 3689 __be32 *p = NULL; 3690 int tidx = idx; 3691 int i, j; 3692 3693 pr_debug("dump wqe at %p\n", mlx5_get_send_wqe(qp, tidx)); 3694 for (i = 0, j = 0; i < size_16 * 4; i += 4, j += 4) { 3695 if ((i & 0xf) == 0) { 3696 void *buf = mlx5_get_send_wqe(qp, tidx); 3697 tidx = (tidx + 1) & (qp->sq.wqe_cnt - 1); 3698 p = buf; 3699 j = 0; 3700 } 3701 pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p[j]), 3702 be32_to_cpu(p[j + 1]), be32_to_cpu(p[j + 2]), 3703 be32_to_cpu(p[j + 3])); 3704 } 3705 } 3706 3707 static void mlx5_bf_copy(u64 __iomem *dst, u64 *src, 3708 unsigned bytecnt, struct mlx5_ib_qp *qp) 3709 { 3710 while (bytecnt > 0) { 3711 __iowrite64_copy(dst++, src++, 8); 3712 __iowrite64_copy(dst++, src++, 8); 3713 __iowrite64_copy(dst++, src++, 8); 3714 __iowrite64_copy(dst++, src++, 8); 3715 __iowrite64_copy(dst++, src++, 8); 3716 __iowrite64_copy(dst++, src++, 8); 3717 __iowrite64_copy(dst++, src++, 8); 3718 __iowrite64_copy(dst++, src++, 8); 3719 bytecnt -= 64; 3720 if (unlikely(src == qp->sq.qend)) 3721 src = mlx5_get_send_wqe(qp, 0); 3722 } 3723 } 3724 3725 static u8 get_fence(u8 fence, struct ib_send_wr *wr) 3726 { 3727 if (unlikely(wr->opcode == IB_WR_LOCAL_INV && 3728 wr->send_flags & IB_SEND_FENCE)) 3729 return MLX5_FENCE_MODE_STRONG_ORDERING; 3730 3731 if (unlikely(fence)) { 3732 if (wr->send_flags & IB_SEND_FENCE) 3733 return MLX5_FENCE_MODE_SMALL_AND_FENCE; 3734 else 3735 return fence; 3736 } else if (unlikely(wr->send_flags & IB_SEND_FENCE)) { 3737 return MLX5_FENCE_MODE_FENCE; 3738 } 3739 3740 return 0; 3741 } 3742 3743 static int begin_wqe(struct mlx5_ib_qp *qp, void **seg, 3744 struct mlx5_wqe_ctrl_seg **ctrl, 3745 struct ib_send_wr *wr, unsigned *idx, 3746 int *size, int nreq) 3747 { 3748 if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq))) 3749 return -ENOMEM; 3750 3751 *idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1); 3752 *seg = mlx5_get_send_wqe(qp, *idx); 3753 *ctrl = *seg; 3754 *(uint32_t *)(*seg + 8) = 0; 3755 (*ctrl)->imm = send_ieth(wr); 3756 (*ctrl)->fm_ce_se = qp->sq_signal_bits | 3757 (wr->send_flags & IB_SEND_SIGNALED ? 3758 MLX5_WQE_CTRL_CQ_UPDATE : 0) | 3759 (wr->send_flags & IB_SEND_SOLICITED ? 3760 MLX5_WQE_CTRL_SOLICITED : 0); 3761 3762 *seg += sizeof(**ctrl); 3763 *size = sizeof(**ctrl) / 16; 3764 3765 return 0; 3766 } 3767 3768 static void finish_wqe(struct mlx5_ib_qp *qp, 3769 struct mlx5_wqe_ctrl_seg *ctrl, 3770 u8 size, unsigned idx, u64 wr_id, 3771 int nreq, u8 fence, u8 next_fence, 3772 u32 mlx5_opcode) 3773 { 3774 u8 opmod = 0; 3775 3776 ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) | 3777 mlx5_opcode | ((u32)opmod << 24)); 3778 ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8)); 3779 ctrl->fm_ce_se |= fence; 3780 qp->fm_cache = next_fence; 3781 if (unlikely(qp->wq_sig)) 3782 ctrl->signature = wq_sig(ctrl); 3783 3784 qp->sq.wrid[idx] = wr_id; 3785 qp->sq.w_list[idx].opcode = mlx5_opcode; 3786 qp->sq.wqe_head[idx] = qp->sq.head + nreq; 3787 qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB); 3788 qp->sq.w_list[idx].next = qp->sq.cur_post; 3789 } 3790 3791 3792 int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 3793 struct ib_send_wr **bad_wr) 3794 { 3795 struct mlx5_wqe_ctrl_seg *ctrl = NULL; /* compiler warning */ 3796 struct mlx5_ib_dev *dev = to_mdev(ibqp->device); 3797 struct mlx5_core_dev *mdev = dev->mdev; 3798 struct mlx5_ib_qp *qp; 3799 struct mlx5_ib_mr *mr; 3800 struct mlx5_wqe_data_seg *dpseg; 3801 struct mlx5_wqe_xrc_seg *xrc; 3802 struct mlx5_bf *bf; 3803 int uninitialized_var(size); 3804 void *qend; 3805 unsigned long flags; 3806 unsigned idx; 3807 int err = 0; 3808 int inl = 0; 3809 int num_sge; 3810 void *seg; 3811 int nreq; 3812 int i; 3813 u8 next_fence = 0; 3814 u8 fence; 3815 3816 if (unlikely(ibqp->qp_type == IB_QPT_GSI)) 3817 return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr); 3818 3819 qp = to_mqp(ibqp); 3820 bf = qp->bf; 3821 qend = qp->sq.qend; 3822 3823 spin_lock_irqsave(&qp->sq.lock, flags); 3824 3825 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 3826 err = -EIO; 3827 *bad_wr = wr; 3828 nreq = 0; 3829 goto out; 3830 } 3831 3832 for (nreq = 0; wr; nreq++, wr = wr->next) { 3833 if (unlikely(wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) { 3834 mlx5_ib_warn(dev, "\n"); 3835 err = -EINVAL; 3836 *bad_wr = wr; 3837 goto out; 3838 } 3839 3840 fence = qp->fm_cache; 3841 num_sge = wr->num_sge; 3842 if (unlikely(num_sge > qp->sq.max_gs)) { 3843 mlx5_ib_warn(dev, "\n"); 3844 err = -EINVAL; 3845 *bad_wr = wr; 3846 goto out; 3847 } 3848 3849 err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, nreq); 3850 if (err) { 3851 mlx5_ib_warn(dev, "\n"); 3852 err = -ENOMEM; 3853 *bad_wr = wr; 3854 goto out; 3855 } 3856 3857 switch (ibqp->qp_type) { 3858 case IB_QPT_XRC_INI: 3859 xrc = seg; 3860 seg += sizeof(*xrc); 3861 size += sizeof(*xrc) / 16; 3862 /* fall through */ 3863 case IB_QPT_RC: 3864 switch (wr->opcode) { 3865 case IB_WR_RDMA_READ: 3866 case IB_WR_RDMA_WRITE: 3867 case IB_WR_RDMA_WRITE_WITH_IMM: 3868 set_raddr_seg(seg, rdma_wr(wr)->remote_addr, 3869 rdma_wr(wr)->rkey); 3870 seg += sizeof(struct mlx5_wqe_raddr_seg); 3871 size += sizeof(struct mlx5_wqe_raddr_seg) / 16; 3872 break; 3873 3874 case IB_WR_ATOMIC_CMP_AND_SWP: 3875 case IB_WR_ATOMIC_FETCH_AND_ADD: 3876 case IB_WR_MASKED_ATOMIC_CMP_AND_SWP: 3877 mlx5_ib_warn(dev, "Atomic operations are not supported yet\n"); 3878 err = -ENOSYS; 3879 *bad_wr = wr; 3880 goto out; 3881 3882 case IB_WR_LOCAL_INV: 3883 next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL; 3884 qp->sq.wr_data[idx] = IB_WR_LOCAL_INV; 3885 ctrl->imm = cpu_to_be32(wr->ex.invalidate_rkey); 3886 set_linv_wr(qp, &seg, &size); 3887 num_sge = 0; 3888 break; 3889 3890 case IB_WR_REG_MR: 3891 next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL; 3892 qp->sq.wr_data[idx] = IB_WR_REG_MR; 3893 ctrl->imm = cpu_to_be32(reg_wr(wr)->key); 3894 err = set_reg_wr(qp, reg_wr(wr), &seg, &size); 3895 if (err) { 3896 *bad_wr = wr; 3897 goto out; 3898 } 3899 num_sge = 0; 3900 break; 3901 3902 case IB_WR_REG_SIG_MR: 3903 qp->sq.wr_data[idx] = IB_WR_REG_SIG_MR; 3904 mr = to_mmr(sig_handover_wr(wr)->sig_mr); 3905 3906 ctrl->imm = cpu_to_be32(mr->ibmr.rkey); 3907 err = set_sig_umr_wr(wr, qp, &seg, &size); 3908 if (err) { 3909 mlx5_ib_warn(dev, "\n"); 3910 *bad_wr = wr; 3911 goto out; 3912 } 3913 3914 finish_wqe(qp, ctrl, size, idx, wr->wr_id, 3915 nreq, get_fence(fence, wr), 3916 next_fence, MLX5_OPCODE_UMR); 3917 /* 3918 * SET_PSV WQEs are not signaled and solicited 3919 * on error 3920 */ 3921 wr->send_flags &= ~IB_SEND_SIGNALED; 3922 wr->send_flags |= IB_SEND_SOLICITED; 3923 err = begin_wqe(qp, &seg, &ctrl, wr, 3924 &idx, &size, nreq); 3925 if (err) { 3926 mlx5_ib_warn(dev, "\n"); 3927 err = -ENOMEM; 3928 *bad_wr = wr; 3929 goto out; 3930 } 3931 3932 err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->mem, 3933 mr->sig->psv_memory.psv_idx, &seg, 3934 &size); 3935 if (err) { 3936 mlx5_ib_warn(dev, "\n"); 3937 *bad_wr = wr; 3938 goto out; 3939 } 3940 3941 finish_wqe(qp, ctrl, size, idx, wr->wr_id, 3942 nreq, get_fence(fence, wr), 3943 next_fence, MLX5_OPCODE_SET_PSV); 3944 err = begin_wqe(qp, &seg, &ctrl, wr, 3945 &idx, &size, nreq); 3946 if (err) { 3947 mlx5_ib_warn(dev, "\n"); 3948 err = -ENOMEM; 3949 *bad_wr = wr; 3950 goto out; 3951 } 3952 3953 next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL; 3954 err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->wire, 3955 mr->sig->psv_wire.psv_idx, &seg, 3956 &size); 3957 if (err) { 3958 mlx5_ib_warn(dev, "\n"); 3959 *bad_wr = wr; 3960 goto out; 3961 } 3962 3963 finish_wqe(qp, ctrl, size, idx, wr->wr_id, 3964 nreq, get_fence(fence, wr), 3965 next_fence, MLX5_OPCODE_SET_PSV); 3966 num_sge = 0; 3967 goto skip_psv; 3968 3969 default: 3970 break; 3971 } 3972 break; 3973 3974 case IB_QPT_UC: 3975 switch (wr->opcode) { 3976 case IB_WR_RDMA_WRITE: 3977 case IB_WR_RDMA_WRITE_WITH_IMM: 3978 set_raddr_seg(seg, rdma_wr(wr)->remote_addr, 3979 rdma_wr(wr)->rkey); 3980 seg += sizeof(struct mlx5_wqe_raddr_seg); 3981 size += sizeof(struct mlx5_wqe_raddr_seg) / 16; 3982 break; 3983 3984 default: 3985 break; 3986 } 3987 break; 3988 3989 case IB_QPT_SMI: 3990 case MLX5_IB_QPT_HW_GSI: 3991 set_datagram_seg(seg, wr); 3992 seg += sizeof(struct mlx5_wqe_datagram_seg); 3993 size += sizeof(struct mlx5_wqe_datagram_seg) / 16; 3994 if (unlikely((seg == qend))) 3995 seg = mlx5_get_send_wqe(qp, 0); 3996 break; 3997 case IB_QPT_UD: 3998 set_datagram_seg(seg, wr); 3999 seg += sizeof(struct mlx5_wqe_datagram_seg); 4000 size += sizeof(struct mlx5_wqe_datagram_seg) / 16; 4001 4002 if (unlikely((seg == qend))) 4003 seg = mlx5_get_send_wqe(qp, 0); 4004 4005 /* handle qp that supports ud offload */ 4006 if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO) { 4007 struct mlx5_wqe_eth_pad *pad; 4008 4009 pad = seg; 4010 memset(pad, 0, sizeof(struct mlx5_wqe_eth_pad)); 4011 seg += sizeof(struct mlx5_wqe_eth_pad); 4012 size += sizeof(struct mlx5_wqe_eth_pad) / 16; 4013 4014 seg = set_eth_seg(seg, wr, qend, qp, &size); 4015 4016 if (unlikely((seg == qend))) 4017 seg = mlx5_get_send_wqe(qp, 0); 4018 } 4019 break; 4020 case MLX5_IB_QPT_REG_UMR: 4021 if (wr->opcode != MLX5_IB_WR_UMR) { 4022 err = -EINVAL; 4023 mlx5_ib_warn(dev, "bad opcode\n"); 4024 goto out; 4025 } 4026 qp->sq.wr_data[idx] = MLX5_IB_WR_UMR; 4027 ctrl->imm = cpu_to_be32(umr_wr(wr)->mkey); 4028 set_reg_umr_segment(seg, wr); 4029 seg += sizeof(struct mlx5_wqe_umr_ctrl_seg); 4030 size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; 4031 if (unlikely((seg == qend))) 4032 seg = mlx5_get_send_wqe(qp, 0); 4033 set_reg_mkey_segment(seg, wr); 4034 seg += sizeof(struct mlx5_mkey_seg); 4035 size += sizeof(struct mlx5_mkey_seg) / 16; 4036 if (unlikely((seg == qend))) 4037 seg = mlx5_get_send_wqe(qp, 0); 4038 break; 4039 4040 default: 4041 break; 4042 } 4043 4044 if (wr->send_flags & IB_SEND_INLINE && num_sge) { 4045 int uninitialized_var(sz); 4046 4047 err = set_data_inl_seg(qp, wr, seg, &sz); 4048 if (unlikely(err)) { 4049 mlx5_ib_warn(dev, "\n"); 4050 *bad_wr = wr; 4051 goto out; 4052 } 4053 inl = 1; 4054 size += sz; 4055 } else { 4056 dpseg = seg; 4057 for (i = 0; i < num_sge; i++) { 4058 if (unlikely(dpseg == qend)) { 4059 seg = mlx5_get_send_wqe(qp, 0); 4060 dpseg = seg; 4061 } 4062 if (likely(wr->sg_list[i].length)) { 4063 set_data_ptr_seg(dpseg, wr->sg_list + i); 4064 size += sizeof(struct mlx5_wqe_data_seg) / 16; 4065 dpseg++; 4066 } 4067 } 4068 } 4069 4070 finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq, 4071 get_fence(fence, wr), next_fence, 4072 mlx5_ib_opcode[wr->opcode]); 4073 skip_psv: 4074 if (0) 4075 dump_wqe(qp, idx, size); 4076 } 4077 4078 out: 4079 if (likely(nreq)) { 4080 qp->sq.head += nreq; 4081 4082 /* Make sure that descriptors are written before 4083 * updating doorbell record and ringing the doorbell 4084 */ 4085 wmb(); 4086 4087 qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post); 4088 4089 /* Make sure doorbell record is visible to the HCA before 4090 * we hit doorbell */ 4091 wmb(); 4092 4093 if (bf->need_lock) 4094 spin_lock(&bf->lock); 4095 else 4096 __acquire(&bf->lock); 4097 4098 /* TBD enable WC */ 4099 if (0 && nreq == 1 && bf->uuarn && inl && size > 1 && size <= bf->buf_size / 16) { 4100 mlx5_bf_copy(bf->reg + bf->offset, (u64 *)ctrl, ALIGN(size * 16, 64), qp); 4101 /* wc_wmb(); */ 4102 } else { 4103 mlx5_write64((__be32 *)ctrl, bf->regreg + bf->offset, 4104 MLX5_GET_DOORBELL_LOCK(&bf->lock32)); 4105 /* Make sure doorbells don't leak out of SQ spinlock 4106 * and reach the HCA out of order. 4107 */ 4108 mmiowb(); 4109 } 4110 bf->offset ^= bf->buf_size; 4111 if (bf->need_lock) 4112 spin_unlock(&bf->lock); 4113 else 4114 __release(&bf->lock); 4115 } 4116 4117 spin_unlock_irqrestore(&qp->sq.lock, flags); 4118 4119 return err; 4120 } 4121 4122 static void set_sig_seg(struct mlx5_rwqe_sig *sig, int size) 4123 { 4124 sig->signature = calc_sig(sig, size); 4125 } 4126 4127 int mlx5_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, 4128 struct ib_recv_wr **bad_wr) 4129 { 4130 struct mlx5_ib_qp *qp = to_mqp(ibqp); 4131 struct mlx5_wqe_data_seg *scat; 4132 struct mlx5_rwqe_sig *sig; 4133 struct mlx5_ib_dev *dev = to_mdev(ibqp->device); 4134 struct mlx5_core_dev *mdev = dev->mdev; 4135 unsigned long flags; 4136 int err = 0; 4137 int nreq; 4138 int ind; 4139 int i; 4140 4141 if (unlikely(ibqp->qp_type == IB_QPT_GSI)) 4142 return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr); 4143 4144 spin_lock_irqsave(&qp->rq.lock, flags); 4145 4146 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 4147 err = -EIO; 4148 *bad_wr = wr; 4149 nreq = 0; 4150 goto out; 4151 } 4152 4153 ind = qp->rq.head & (qp->rq.wqe_cnt - 1); 4154 4155 for (nreq = 0; wr; nreq++, wr = wr->next) { 4156 if (mlx5_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) { 4157 err = -ENOMEM; 4158 *bad_wr = wr; 4159 goto out; 4160 } 4161 4162 if (unlikely(wr->num_sge > qp->rq.max_gs)) { 4163 err = -EINVAL; 4164 *bad_wr = wr; 4165 goto out; 4166 } 4167 4168 scat = get_recv_wqe(qp, ind); 4169 if (qp->wq_sig) 4170 scat++; 4171 4172 for (i = 0; i < wr->num_sge; i++) 4173 set_data_ptr_seg(scat + i, wr->sg_list + i); 4174 4175 if (i < qp->rq.max_gs) { 4176 scat[i].byte_count = 0; 4177 scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY); 4178 scat[i].addr = 0; 4179 } 4180 4181 if (qp->wq_sig) { 4182 sig = (struct mlx5_rwqe_sig *)scat; 4183 set_sig_seg(sig, (qp->rq.max_gs + 1) << 2); 4184 } 4185 4186 qp->rq.wrid[ind] = wr->wr_id; 4187 4188 ind = (ind + 1) & (qp->rq.wqe_cnt - 1); 4189 } 4190 4191 out: 4192 if (likely(nreq)) { 4193 qp->rq.head += nreq; 4194 4195 /* Make sure that descriptors are written before 4196 * doorbell record. 4197 */ 4198 wmb(); 4199 4200 *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff); 4201 } 4202 4203 spin_unlock_irqrestore(&qp->rq.lock, flags); 4204 4205 return err; 4206 } 4207 4208 static inline enum ib_qp_state to_ib_qp_state(enum mlx5_qp_state mlx5_state) 4209 { 4210 switch (mlx5_state) { 4211 case MLX5_QP_STATE_RST: return IB_QPS_RESET; 4212 case MLX5_QP_STATE_INIT: return IB_QPS_INIT; 4213 case MLX5_QP_STATE_RTR: return IB_QPS_RTR; 4214 case MLX5_QP_STATE_RTS: return IB_QPS_RTS; 4215 case MLX5_QP_STATE_SQ_DRAINING: 4216 case MLX5_QP_STATE_SQD: return IB_QPS_SQD; 4217 case MLX5_QP_STATE_SQER: return IB_QPS_SQE; 4218 case MLX5_QP_STATE_ERR: return IB_QPS_ERR; 4219 default: return -1; 4220 } 4221 } 4222 4223 static inline enum ib_mig_state to_ib_mig_state(int mlx5_mig_state) 4224 { 4225 switch (mlx5_mig_state) { 4226 case MLX5_QP_PM_ARMED: return IB_MIG_ARMED; 4227 case MLX5_QP_PM_REARM: return IB_MIG_REARM; 4228 case MLX5_QP_PM_MIGRATED: return IB_MIG_MIGRATED; 4229 default: return -1; 4230 } 4231 } 4232 4233 static int to_ib_qp_access_flags(int mlx5_flags) 4234 { 4235 int ib_flags = 0; 4236 4237 if (mlx5_flags & MLX5_QP_BIT_RRE) 4238 ib_flags |= IB_ACCESS_REMOTE_READ; 4239 if (mlx5_flags & MLX5_QP_BIT_RWE) 4240 ib_flags |= IB_ACCESS_REMOTE_WRITE; 4241 if (mlx5_flags & MLX5_QP_BIT_RAE) 4242 ib_flags |= IB_ACCESS_REMOTE_ATOMIC; 4243 4244 return ib_flags; 4245 } 4246 4247 static void to_ib_ah_attr(struct mlx5_ib_dev *ibdev, struct ib_ah_attr *ib_ah_attr, 4248 struct mlx5_qp_path *path) 4249 { 4250 struct mlx5_core_dev *dev = ibdev->mdev; 4251 4252 memset(ib_ah_attr, 0, sizeof(*ib_ah_attr)); 4253 ib_ah_attr->port_num = path->port; 4254 4255 if (ib_ah_attr->port_num == 0 || 4256 ib_ah_attr->port_num > MLX5_CAP_GEN(dev, num_ports)) 4257 return; 4258 4259 ib_ah_attr->sl = path->dci_cfi_prio_sl & 0xf; 4260 4261 ib_ah_attr->dlid = be16_to_cpu(path->rlid); 4262 ib_ah_attr->src_path_bits = path->grh_mlid & 0x7f; 4263 ib_ah_attr->static_rate = path->static_rate ? path->static_rate - 5 : 0; 4264 ib_ah_attr->ah_flags = (path->grh_mlid & (1 << 7)) ? IB_AH_GRH : 0; 4265 if (ib_ah_attr->ah_flags) { 4266 ib_ah_attr->grh.sgid_index = path->mgid_index; 4267 ib_ah_attr->grh.hop_limit = path->hop_limit; 4268 ib_ah_attr->grh.traffic_class = 4269 (be32_to_cpu(path->tclass_flowlabel) >> 20) & 0xff; 4270 ib_ah_attr->grh.flow_label = 4271 be32_to_cpu(path->tclass_flowlabel) & 0xfffff; 4272 memcpy(ib_ah_attr->grh.dgid.raw, 4273 path->rgid, sizeof(ib_ah_attr->grh.dgid.raw)); 4274 } 4275 } 4276 4277 static int query_raw_packet_qp_sq_state(struct mlx5_ib_dev *dev, 4278 struct mlx5_ib_sq *sq, 4279 u8 *sq_state) 4280 { 4281 void *out; 4282 void *sqc; 4283 int inlen; 4284 int err; 4285 4286 inlen = MLX5_ST_SZ_BYTES(query_sq_out); 4287 out = mlx5_vzalloc(inlen); 4288 if (!out) 4289 return -ENOMEM; 4290 4291 err = mlx5_core_query_sq(dev->mdev, sq->base.mqp.qpn, out); 4292 if (err) 4293 goto out; 4294 4295 sqc = MLX5_ADDR_OF(query_sq_out, out, sq_context); 4296 *sq_state = MLX5_GET(sqc, sqc, state); 4297 sq->state = *sq_state; 4298 4299 out: 4300 kvfree(out); 4301 return err; 4302 } 4303 4304 static int query_raw_packet_qp_rq_state(struct mlx5_ib_dev *dev, 4305 struct mlx5_ib_rq *rq, 4306 u8 *rq_state) 4307 { 4308 void *out; 4309 void *rqc; 4310 int inlen; 4311 int err; 4312 4313 inlen = MLX5_ST_SZ_BYTES(query_rq_out); 4314 out = mlx5_vzalloc(inlen); 4315 if (!out) 4316 return -ENOMEM; 4317 4318 err = mlx5_core_query_rq(dev->mdev, rq->base.mqp.qpn, out); 4319 if (err) 4320 goto out; 4321 4322 rqc = MLX5_ADDR_OF(query_rq_out, out, rq_context); 4323 *rq_state = MLX5_GET(rqc, rqc, state); 4324 rq->state = *rq_state; 4325 4326 out: 4327 kvfree(out); 4328 return err; 4329 } 4330 4331 static int sqrq_state_to_qp_state(u8 sq_state, u8 rq_state, 4332 struct mlx5_ib_qp *qp, u8 *qp_state) 4333 { 4334 static const u8 sqrq_trans[MLX5_RQ_NUM_STATE][MLX5_SQ_NUM_STATE] = { 4335 [MLX5_RQC_STATE_RST] = { 4336 [MLX5_SQC_STATE_RST] = IB_QPS_RESET, 4337 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE_BAD, 4338 [MLX5_SQC_STATE_ERR] = MLX5_QP_STATE_BAD, 4339 [MLX5_SQ_STATE_NA] = IB_QPS_RESET, 4340 }, 4341 [MLX5_RQC_STATE_RDY] = { 4342 [MLX5_SQC_STATE_RST] = MLX5_QP_STATE_BAD, 4343 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE, 4344 [MLX5_SQC_STATE_ERR] = IB_QPS_SQE, 4345 [MLX5_SQ_STATE_NA] = MLX5_QP_STATE, 4346 }, 4347 [MLX5_RQC_STATE_ERR] = { 4348 [MLX5_SQC_STATE_RST] = MLX5_QP_STATE_BAD, 4349 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE_BAD, 4350 [MLX5_SQC_STATE_ERR] = IB_QPS_ERR, 4351 [MLX5_SQ_STATE_NA] = IB_QPS_ERR, 4352 }, 4353 [MLX5_RQ_STATE_NA] = { 4354 [MLX5_SQC_STATE_RST] = IB_QPS_RESET, 4355 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE, 4356 [MLX5_SQC_STATE_ERR] = MLX5_QP_STATE, 4357 [MLX5_SQ_STATE_NA] = MLX5_QP_STATE_BAD, 4358 }, 4359 }; 4360 4361 *qp_state = sqrq_trans[rq_state][sq_state]; 4362 4363 if (*qp_state == MLX5_QP_STATE_BAD) { 4364 WARN(1, "Buggy Raw Packet QP state, SQ 0x%x state: 0x%x, RQ 0x%x state: 0x%x", 4365 qp->raw_packet_qp.sq.base.mqp.qpn, sq_state, 4366 qp->raw_packet_qp.rq.base.mqp.qpn, rq_state); 4367 return -EINVAL; 4368 } 4369 4370 if (*qp_state == MLX5_QP_STATE) 4371 *qp_state = qp->state; 4372 4373 return 0; 4374 } 4375 4376 static int query_raw_packet_qp_state(struct mlx5_ib_dev *dev, 4377 struct mlx5_ib_qp *qp, 4378 u8 *raw_packet_qp_state) 4379 { 4380 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp; 4381 struct mlx5_ib_sq *sq = &raw_packet_qp->sq; 4382 struct mlx5_ib_rq *rq = &raw_packet_qp->rq; 4383 int err; 4384 u8 sq_state = MLX5_SQ_STATE_NA; 4385 u8 rq_state = MLX5_RQ_STATE_NA; 4386 4387 if (qp->sq.wqe_cnt) { 4388 err = query_raw_packet_qp_sq_state(dev, sq, &sq_state); 4389 if (err) 4390 return err; 4391 } 4392 4393 if (qp->rq.wqe_cnt) { 4394 err = query_raw_packet_qp_rq_state(dev, rq, &rq_state); 4395 if (err) 4396 return err; 4397 } 4398 4399 return sqrq_state_to_qp_state(sq_state, rq_state, qp, 4400 raw_packet_qp_state); 4401 } 4402 4403 static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, 4404 struct ib_qp_attr *qp_attr) 4405 { 4406 int outlen = MLX5_ST_SZ_BYTES(query_qp_out); 4407 struct mlx5_qp_context *context; 4408 int mlx5_state; 4409 u32 *outb; 4410 int err = 0; 4411 4412 outb = kzalloc(outlen, GFP_KERNEL); 4413 if (!outb) 4414 return -ENOMEM; 4415 4416 err = mlx5_core_qp_query(dev->mdev, &qp->trans_qp.base.mqp, outb, 4417 outlen); 4418 if (err) 4419 goto out; 4420 4421 /* FIXME: use MLX5_GET rather than mlx5_qp_context manual struct */ 4422 context = (struct mlx5_qp_context *)MLX5_ADDR_OF(query_qp_out, outb, qpc); 4423 4424 mlx5_state = be32_to_cpu(context->flags) >> 28; 4425 4426 qp->state = to_ib_qp_state(mlx5_state); 4427 qp_attr->path_mtu = context->mtu_msgmax >> 5; 4428 qp_attr->path_mig_state = 4429 to_ib_mig_state((be32_to_cpu(context->flags) >> 11) & 0x3); 4430 qp_attr->qkey = be32_to_cpu(context->qkey); 4431 qp_attr->rq_psn = be32_to_cpu(context->rnr_nextrecvpsn) & 0xffffff; 4432 qp_attr->sq_psn = be32_to_cpu(context->next_send_psn) & 0xffffff; 4433 qp_attr->dest_qp_num = be32_to_cpu(context->log_pg_sz_remote_qpn) & 0xffffff; 4434 qp_attr->qp_access_flags = 4435 to_ib_qp_access_flags(be32_to_cpu(context->params2)); 4436 4437 if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) { 4438 to_ib_ah_attr(dev, &qp_attr->ah_attr, &context->pri_path); 4439 to_ib_ah_attr(dev, &qp_attr->alt_ah_attr, &context->alt_path); 4440 qp_attr->alt_pkey_index = 4441 be16_to_cpu(context->alt_path.pkey_index); 4442 qp_attr->alt_port_num = qp_attr->alt_ah_attr.port_num; 4443 } 4444 4445 qp_attr->pkey_index = be16_to_cpu(context->pri_path.pkey_index); 4446 qp_attr->port_num = context->pri_path.port; 4447 4448 /* qp_attr->en_sqd_async_notify is only applicable in modify qp */ 4449 qp_attr->sq_draining = mlx5_state == MLX5_QP_STATE_SQ_DRAINING; 4450 4451 qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context->params1) >> 21) & 0x7); 4452 4453 qp_attr->max_dest_rd_atomic = 4454 1 << ((be32_to_cpu(context->params2) >> 21) & 0x7); 4455 qp_attr->min_rnr_timer = 4456 (be32_to_cpu(context->rnr_nextrecvpsn) >> 24) & 0x1f; 4457 qp_attr->timeout = context->pri_path.ackto_lt >> 3; 4458 qp_attr->retry_cnt = (be32_to_cpu(context->params1) >> 16) & 0x7; 4459 qp_attr->rnr_retry = (be32_to_cpu(context->params1) >> 13) & 0x7; 4460 qp_attr->alt_timeout = context->alt_path.ackto_lt >> 3; 4461 4462 out: 4463 kfree(outb); 4464 return err; 4465 } 4466 4467 int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 4468 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) 4469 { 4470 struct mlx5_ib_dev *dev = to_mdev(ibqp->device); 4471 struct mlx5_ib_qp *qp = to_mqp(ibqp); 4472 int err = 0; 4473 u8 raw_packet_qp_state; 4474 4475 if (ibqp->rwq_ind_tbl) 4476 return -ENOSYS; 4477 4478 if (unlikely(ibqp->qp_type == IB_QPT_GSI)) 4479 return mlx5_ib_gsi_query_qp(ibqp, qp_attr, qp_attr_mask, 4480 qp_init_attr); 4481 4482 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 4483 /* 4484 * Wait for any outstanding page faults, in case the user frees memory 4485 * based upon this query's result. 4486 */ 4487 flush_workqueue(mlx5_ib_page_fault_wq); 4488 #endif 4489 4490 mutex_lock(&qp->mutex); 4491 4492 if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET) { 4493 err = query_raw_packet_qp_state(dev, qp, &raw_packet_qp_state); 4494 if (err) 4495 goto out; 4496 qp->state = raw_packet_qp_state; 4497 qp_attr->port_num = 1; 4498 } else { 4499 err = query_qp_attr(dev, qp, qp_attr); 4500 if (err) 4501 goto out; 4502 } 4503 4504 qp_attr->qp_state = qp->state; 4505 qp_attr->cur_qp_state = qp_attr->qp_state; 4506 qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt; 4507 qp_attr->cap.max_recv_sge = qp->rq.max_gs; 4508 4509 if (!ibqp->uobject) { 4510 qp_attr->cap.max_send_wr = qp->sq.max_post; 4511 qp_attr->cap.max_send_sge = qp->sq.max_gs; 4512 qp_init_attr->qp_context = ibqp->qp_context; 4513 } else { 4514 qp_attr->cap.max_send_wr = 0; 4515 qp_attr->cap.max_send_sge = 0; 4516 } 4517 4518 qp_init_attr->qp_type = ibqp->qp_type; 4519 qp_init_attr->recv_cq = ibqp->recv_cq; 4520 qp_init_attr->send_cq = ibqp->send_cq; 4521 qp_init_attr->srq = ibqp->srq; 4522 qp_attr->cap.max_inline_data = qp->max_inline_data; 4523 4524 qp_init_attr->cap = qp_attr->cap; 4525 4526 qp_init_attr->create_flags = 0; 4527 if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK) 4528 qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK; 4529 4530 if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL) 4531 qp_init_attr->create_flags |= IB_QP_CREATE_CROSS_CHANNEL; 4532 if (qp->flags & MLX5_IB_QP_MANAGED_SEND) 4533 qp_init_attr->create_flags |= IB_QP_CREATE_MANAGED_SEND; 4534 if (qp->flags & MLX5_IB_QP_MANAGED_RECV) 4535 qp_init_attr->create_flags |= IB_QP_CREATE_MANAGED_RECV; 4536 if (qp->flags & MLX5_IB_QP_SQPN_QP1) 4537 qp_init_attr->create_flags |= mlx5_ib_create_qp_sqpn_qp1(); 4538 4539 qp_init_attr->sq_sig_type = qp->sq_signal_bits & MLX5_WQE_CTRL_CQ_UPDATE ? 4540 IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; 4541 4542 out: 4543 mutex_unlock(&qp->mutex); 4544 return err; 4545 } 4546 4547 struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev, 4548 struct ib_ucontext *context, 4549 struct ib_udata *udata) 4550 { 4551 struct mlx5_ib_dev *dev = to_mdev(ibdev); 4552 struct mlx5_ib_xrcd *xrcd; 4553 int err; 4554 4555 if (!MLX5_CAP_GEN(dev->mdev, xrc)) 4556 return ERR_PTR(-ENOSYS); 4557 4558 xrcd = kmalloc(sizeof(*xrcd), GFP_KERNEL); 4559 if (!xrcd) 4560 return ERR_PTR(-ENOMEM); 4561 4562 err = mlx5_core_xrcd_alloc(dev->mdev, &xrcd->xrcdn); 4563 if (err) { 4564 kfree(xrcd); 4565 return ERR_PTR(-ENOMEM); 4566 } 4567 4568 return &xrcd->ibxrcd; 4569 } 4570 4571 int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd) 4572 { 4573 struct mlx5_ib_dev *dev = to_mdev(xrcd->device); 4574 u32 xrcdn = to_mxrcd(xrcd)->xrcdn; 4575 int err; 4576 4577 err = mlx5_core_xrcd_dealloc(dev->mdev, xrcdn); 4578 if (err) { 4579 mlx5_ib_warn(dev, "failed to dealloc xrcdn 0x%x\n", xrcdn); 4580 return err; 4581 } 4582 4583 kfree(xrcd); 4584 4585 return 0; 4586 } 4587 4588 static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type) 4589 { 4590 struct mlx5_ib_rwq *rwq = to_mibrwq(core_qp); 4591 struct mlx5_ib_dev *dev = to_mdev(rwq->ibwq.device); 4592 struct ib_event event; 4593 4594 if (rwq->ibwq.event_handler) { 4595 event.device = rwq->ibwq.device; 4596 event.element.wq = &rwq->ibwq; 4597 switch (type) { 4598 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR: 4599 event.event = IB_EVENT_WQ_FATAL; 4600 break; 4601 default: 4602 mlx5_ib_warn(dev, "Unexpected event type %d on WQ %06x\n", type, core_qp->qpn); 4603 return; 4604 } 4605 4606 rwq->ibwq.event_handler(&event, rwq->ibwq.wq_context); 4607 } 4608 } 4609 4610 static int create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd, 4611 struct ib_wq_init_attr *init_attr) 4612 { 4613 struct mlx5_ib_dev *dev; 4614 __be64 *rq_pas0; 4615 void *in; 4616 void *rqc; 4617 void *wq; 4618 int inlen; 4619 int err; 4620 4621 dev = to_mdev(pd->device); 4622 4623 inlen = MLX5_ST_SZ_BYTES(create_rq_in) + sizeof(u64) * rwq->rq_num_pas; 4624 in = mlx5_vzalloc(inlen); 4625 if (!in) 4626 return -ENOMEM; 4627 4628 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); 4629 MLX5_SET(rqc, rqc, mem_rq_type, 4630 MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE); 4631 MLX5_SET(rqc, rqc, user_index, rwq->user_index); 4632 MLX5_SET(rqc, rqc, cqn, to_mcq(init_attr->cq)->mcq.cqn); 4633 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); 4634 MLX5_SET(rqc, rqc, flush_in_error_en, 1); 4635 wq = MLX5_ADDR_OF(rqc, rqc, wq); 4636 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); 4637 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN); 4638 MLX5_SET(wq, wq, log_wq_stride, rwq->log_rq_stride); 4639 MLX5_SET(wq, wq, log_wq_sz, rwq->log_rq_size); 4640 MLX5_SET(wq, wq, pd, to_mpd(pd)->pdn); 4641 MLX5_SET(wq, wq, page_offset, rwq->rq_page_offset); 4642 MLX5_SET(wq, wq, log_wq_pg_sz, rwq->log_page_size); 4643 MLX5_SET(wq, wq, wq_signature, rwq->wq_sig); 4644 MLX5_SET64(wq, wq, dbr_addr, rwq->db.dma); 4645 rq_pas0 = (__be64 *)MLX5_ADDR_OF(wq, wq, pas); 4646 mlx5_ib_populate_pas(dev, rwq->umem, rwq->page_shift, rq_pas0, 0); 4647 err = mlx5_core_create_rq_tracked(dev->mdev, in, inlen, &rwq->core_qp); 4648 kvfree(in); 4649 return err; 4650 } 4651 4652 static int set_user_rq_size(struct mlx5_ib_dev *dev, 4653 struct ib_wq_init_attr *wq_init_attr, 4654 struct mlx5_ib_create_wq *ucmd, 4655 struct mlx5_ib_rwq *rwq) 4656 { 4657 /* Sanity check RQ size before proceeding */ 4658 if (wq_init_attr->max_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_wq_sz))) 4659 return -EINVAL; 4660 4661 if (!ucmd->rq_wqe_count) 4662 return -EINVAL; 4663 4664 rwq->wqe_count = ucmd->rq_wqe_count; 4665 rwq->wqe_shift = ucmd->rq_wqe_shift; 4666 rwq->buf_size = (rwq->wqe_count << rwq->wqe_shift); 4667 rwq->log_rq_stride = rwq->wqe_shift; 4668 rwq->log_rq_size = ilog2(rwq->wqe_count); 4669 return 0; 4670 } 4671 4672 static int prepare_user_rq(struct ib_pd *pd, 4673 struct ib_wq_init_attr *init_attr, 4674 struct ib_udata *udata, 4675 struct mlx5_ib_rwq *rwq) 4676 { 4677 struct mlx5_ib_dev *dev = to_mdev(pd->device); 4678 struct mlx5_ib_create_wq ucmd = {}; 4679 int err; 4680 size_t required_cmd_sz; 4681 4682 required_cmd_sz = offsetof(typeof(ucmd), reserved) + sizeof(ucmd.reserved); 4683 if (udata->inlen < required_cmd_sz) { 4684 mlx5_ib_dbg(dev, "invalid inlen\n"); 4685 return -EINVAL; 4686 } 4687 4688 if (udata->inlen > sizeof(ucmd) && 4689 !ib_is_udata_cleared(udata, sizeof(ucmd), 4690 udata->inlen - sizeof(ucmd))) { 4691 mlx5_ib_dbg(dev, "inlen is not supported\n"); 4692 return -EOPNOTSUPP; 4693 } 4694 4695 if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) { 4696 mlx5_ib_dbg(dev, "copy failed\n"); 4697 return -EFAULT; 4698 } 4699 4700 if (ucmd.comp_mask) { 4701 mlx5_ib_dbg(dev, "invalid comp mask\n"); 4702 return -EOPNOTSUPP; 4703 } 4704 4705 if (ucmd.reserved) { 4706 mlx5_ib_dbg(dev, "invalid reserved\n"); 4707 return -EOPNOTSUPP; 4708 } 4709 4710 err = set_user_rq_size(dev, init_attr, &ucmd, rwq); 4711 if (err) { 4712 mlx5_ib_dbg(dev, "err %d\n", err); 4713 return err; 4714 } 4715 4716 err = create_user_rq(dev, pd, rwq, &ucmd); 4717 if (err) { 4718 mlx5_ib_dbg(dev, "err %d\n", err); 4719 if (err) 4720 return err; 4721 } 4722 4723 rwq->user_index = ucmd.user_index; 4724 return 0; 4725 } 4726 4727 struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd, 4728 struct ib_wq_init_attr *init_attr, 4729 struct ib_udata *udata) 4730 { 4731 struct mlx5_ib_dev *dev; 4732 struct mlx5_ib_rwq *rwq; 4733 struct mlx5_ib_create_wq_resp resp = {}; 4734 size_t min_resp_len; 4735 int err; 4736 4737 if (!udata) 4738 return ERR_PTR(-ENOSYS); 4739 4740 min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved); 4741 if (udata->outlen && udata->outlen < min_resp_len) 4742 return ERR_PTR(-EINVAL); 4743 4744 dev = to_mdev(pd->device); 4745 switch (init_attr->wq_type) { 4746 case IB_WQT_RQ: 4747 rwq = kzalloc(sizeof(*rwq), GFP_KERNEL); 4748 if (!rwq) 4749 return ERR_PTR(-ENOMEM); 4750 err = prepare_user_rq(pd, init_attr, udata, rwq); 4751 if (err) 4752 goto err; 4753 err = create_rq(rwq, pd, init_attr); 4754 if (err) 4755 goto err_user_rq; 4756 break; 4757 default: 4758 mlx5_ib_dbg(dev, "unsupported wq type %d\n", 4759 init_attr->wq_type); 4760 return ERR_PTR(-EINVAL); 4761 } 4762 4763 rwq->ibwq.wq_num = rwq->core_qp.qpn; 4764 rwq->ibwq.state = IB_WQS_RESET; 4765 if (udata->outlen) { 4766 resp.response_length = offsetof(typeof(resp), response_length) + 4767 sizeof(resp.response_length); 4768 err = ib_copy_to_udata(udata, &resp, resp.response_length); 4769 if (err) 4770 goto err_copy; 4771 } 4772 4773 rwq->core_qp.event = mlx5_ib_wq_event; 4774 rwq->ibwq.event_handler = init_attr->event_handler; 4775 return &rwq->ibwq; 4776 4777 err_copy: 4778 mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp); 4779 err_user_rq: 4780 destroy_user_rq(pd, rwq); 4781 err: 4782 kfree(rwq); 4783 return ERR_PTR(err); 4784 } 4785 4786 int mlx5_ib_destroy_wq(struct ib_wq *wq) 4787 { 4788 struct mlx5_ib_dev *dev = to_mdev(wq->device); 4789 struct mlx5_ib_rwq *rwq = to_mrwq(wq); 4790 4791 mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp); 4792 destroy_user_rq(wq->pd, rwq); 4793 kfree(rwq); 4794 4795 return 0; 4796 } 4797 4798 struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device, 4799 struct ib_rwq_ind_table_init_attr *init_attr, 4800 struct ib_udata *udata) 4801 { 4802 struct mlx5_ib_dev *dev = to_mdev(device); 4803 struct mlx5_ib_rwq_ind_table *rwq_ind_tbl; 4804 int sz = 1 << init_attr->log_ind_tbl_size; 4805 struct mlx5_ib_create_rwq_ind_tbl_resp resp = {}; 4806 size_t min_resp_len; 4807 int inlen; 4808 int err; 4809 int i; 4810 u32 *in; 4811 void *rqtc; 4812 4813 if (udata->inlen > 0 && 4814 !ib_is_udata_cleared(udata, 0, 4815 udata->inlen)) 4816 return ERR_PTR(-EOPNOTSUPP); 4817 4818 min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved); 4819 if (udata->outlen && udata->outlen < min_resp_len) 4820 return ERR_PTR(-EINVAL); 4821 4822 rwq_ind_tbl = kzalloc(sizeof(*rwq_ind_tbl), GFP_KERNEL); 4823 if (!rwq_ind_tbl) 4824 return ERR_PTR(-ENOMEM); 4825 4826 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz; 4827 in = mlx5_vzalloc(inlen); 4828 if (!in) { 4829 err = -ENOMEM; 4830 goto err; 4831 } 4832 4833 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); 4834 4835 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz); 4836 MLX5_SET(rqtc, rqtc, rqt_max_size, sz); 4837 4838 for (i = 0; i < sz; i++) 4839 MLX5_SET(rqtc, rqtc, rq_num[i], init_attr->ind_tbl[i]->wq_num); 4840 4841 err = mlx5_core_create_rqt(dev->mdev, in, inlen, &rwq_ind_tbl->rqtn); 4842 kvfree(in); 4843 4844 if (err) 4845 goto err; 4846 4847 rwq_ind_tbl->ib_rwq_ind_tbl.ind_tbl_num = rwq_ind_tbl->rqtn; 4848 if (udata->outlen) { 4849 resp.response_length = offsetof(typeof(resp), response_length) + 4850 sizeof(resp.response_length); 4851 err = ib_copy_to_udata(udata, &resp, resp.response_length); 4852 if (err) 4853 goto err_copy; 4854 } 4855 4856 return &rwq_ind_tbl->ib_rwq_ind_tbl; 4857 4858 err_copy: 4859 mlx5_core_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn); 4860 err: 4861 kfree(rwq_ind_tbl); 4862 return ERR_PTR(err); 4863 } 4864 4865 int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl) 4866 { 4867 struct mlx5_ib_rwq_ind_table *rwq_ind_tbl = to_mrwq_ind_table(ib_rwq_ind_tbl); 4868 struct mlx5_ib_dev *dev = to_mdev(ib_rwq_ind_tbl->device); 4869 4870 mlx5_core_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn); 4871 4872 kfree(rwq_ind_tbl); 4873 return 0; 4874 } 4875 4876 int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr, 4877 u32 wq_attr_mask, struct ib_udata *udata) 4878 { 4879 struct mlx5_ib_dev *dev = to_mdev(wq->device); 4880 struct mlx5_ib_rwq *rwq = to_mrwq(wq); 4881 struct mlx5_ib_modify_wq ucmd = {}; 4882 size_t required_cmd_sz; 4883 int curr_wq_state; 4884 int wq_state; 4885 int inlen; 4886 int err; 4887 void *rqc; 4888 void *in; 4889 4890 required_cmd_sz = offsetof(typeof(ucmd), reserved) + sizeof(ucmd.reserved); 4891 if (udata->inlen < required_cmd_sz) 4892 return -EINVAL; 4893 4894 if (udata->inlen > sizeof(ucmd) && 4895 !ib_is_udata_cleared(udata, sizeof(ucmd), 4896 udata->inlen - sizeof(ucmd))) 4897 return -EOPNOTSUPP; 4898 4899 if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) 4900 return -EFAULT; 4901 4902 if (ucmd.comp_mask || ucmd.reserved) 4903 return -EOPNOTSUPP; 4904 4905 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 4906 in = mlx5_vzalloc(inlen); 4907 if (!in) 4908 return -ENOMEM; 4909 4910 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 4911 4912 curr_wq_state = (wq_attr_mask & IB_WQ_CUR_STATE) ? 4913 wq_attr->curr_wq_state : wq->state; 4914 wq_state = (wq_attr_mask & IB_WQ_STATE) ? 4915 wq_attr->wq_state : curr_wq_state; 4916 if (curr_wq_state == IB_WQS_ERR) 4917 curr_wq_state = MLX5_RQC_STATE_ERR; 4918 if (wq_state == IB_WQS_ERR) 4919 wq_state = MLX5_RQC_STATE_ERR; 4920 MLX5_SET(modify_rq_in, in, rq_state, curr_wq_state); 4921 MLX5_SET(rqc, rqc, state, wq_state); 4922 4923 err = mlx5_core_modify_rq(dev->mdev, rwq->core_qp.qpn, in, inlen); 4924 kvfree(in); 4925 if (!err) 4926 rwq->ibwq.state = (wq_state == MLX5_RQC_STATE_ERR) ? IB_WQS_ERR : wq_state; 4927 4928 return err; 4929 } 4930