1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <linux/gfp.h> 7 #include <linux/mlx5/qp.h> 8 #include <linux/mlx5/driver.h> 9 #include "wr.h" 10 11 static const u32 mlx5_ib_opcode[] = { 12 [IB_WR_SEND] = MLX5_OPCODE_SEND, 13 [IB_WR_LSO] = MLX5_OPCODE_LSO, 14 [IB_WR_SEND_WITH_IMM] = MLX5_OPCODE_SEND_IMM, 15 [IB_WR_RDMA_WRITE] = MLX5_OPCODE_RDMA_WRITE, 16 [IB_WR_RDMA_WRITE_WITH_IMM] = MLX5_OPCODE_RDMA_WRITE_IMM, 17 [IB_WR_RDMA_READ] = MLX5_OPCODE_RDMA_READ, 18 [IB_WR_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_CS, 19 [IB_WR_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_FA, 20 [IB_WR_SEND_WITH_INV] = MLX5_OPCODE_SEND_INVAL, 21 [IB_WR_LOCAL_INV] = MLX5_OPCODE_UMR, 22 [IB_WR_REG_MR] = MLX5_OPCODE_UMR, 23 [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_MASKED_CS, 24 [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_MASKED_FA, 25 [MLX5_IB_WR_UMR] = MLX5_OPCODE_UMR, 26 }; 27 28 /* handle_post_send_edge - Check if we get to SQ edge. If yes, update to the 29 * next nearby edge and get new address translation for current WQE position. 30 * @sq - SQ buffer. 31 * @seg: Current WQE position (16B aligned). 32 * @wqe_sz: Total current WQE size [16B]. 33 * @cur_edge: Updated current edge. 34 */ 35 static inline void handle_post_send_edge(struct mlx5_ib_wq *sq, void **seg, 36 u32 wqe_sz, void **cur_edge) 37 { 38 u32 idx; 39 40 if (likely(*seg != *cur_edge)) 41 return; 42 43 idx = (sq->cur_post + (wqe_sz >> 2)) & (sq->wqe_cnt - 1); 44 *cur_edge = get_sq_edge(sq, idx); 45 46 *seg = mlx5_frag_buf_get_wqe(&sq->fbc, idx); 47 } 48 49 /* memcpy_send_wqe - copy data from src to WQE and update the relevant WQ's 50 * pointers. At the end @seg is aligned to 16B regardless the copied size. 51 * @sq - SQ buffer. 52 * @cur_edge: Updated current edge. 53 * @seg: Current WQE position (16B aligned). 54 * @wqe_sz: Total current WQE size [16B]. 55 * @src: Pointer to copy from. 56 * @n: Number of bytes to copy. 57 */ 58 static inline void memcpy_send_wqe(struct mlx5_ib_wq *sq, void **cur_edge, 59 void **seg, u32 *wqe_sz, const void *src, 60 size_t n) 61 { 62 while (likely(n)) { 63 size_t leftlen = *cur_edge - *seg; 64 size_t copysz = min_t(size_t, leftlen, n); 65 size_t stride; 66 67 memcpy(*seg, src, copysz); 68 69 n -= copysz; 70 src += copysz; 71 stride = !n ? ALIGN(copysz, 16) : copysz; 72 *seg += stride; 73 *wqe_sz += stride >> 4; 74 handle_post_send_edge(sq, seg, *wqe_sz, cur_edge); 75 } 76 } 77 78 static int mlx5_wq_overflow(struct mlx5_ib_wq *wq, int nreq, 79 struct ib_cq *ib_cq) 80 { 81 struct mlx5_ib_cq *cq; 82 unsigned int cur; 83 84 cur = wq->head - wq->tail; 85 if (likely(cur + nreq < wq->max_post)) 86 return 0; 87 88 cq = to_mcq(ib_cq); 89 spin_lock(&cq->lock); 90 cur = wq->head - wq->tail; 91 spin_unlock(&cq->lock); 92 93 return cur + nreq >= wq->max_post; 94 } 95 96 static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg, 97 u64 remote_addr, u32 rkey) 98 { 99 rseg->raddr = cpu_to_be64(remote_addr); 100 rseg->rkey = cpu_to_be32(rkey); 101 rseg->reserved = 0; 102 } 103 104 static void set_eth_seg(const struct ib_send_wr *wr, struct mlx5_ib_qp *qp, 105 void **seg, int *size, void **cur_edge) 106 { 107 struct mlx5_wqe_eth_seg *eseg = *seg; 108 109 memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg)); 110 111 if (wr->send_flags & IB_SEND_IP_CSUM) 112 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM | 113 MLX5_ETH_WQE_L4_CSUM; 114 115 if (wr->opcode == IB_WR_LSO) { 116 struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr); 117 size_t left, copysz; 118 void *pdata = ud_wr->header; 119 size_t stride; 120 121 left = ud_wr->hlen; 122 eseg->mss = cpu_to_be16(ud_wr->mss); 123 eseg->inline_hdr.sz = cpu_to_be16(left); 124 125 /* memcpy_send_wqe should get a 16B align address. Hence, we 126 * first copy up to the current edge and then, if needed, 127 * continue to memcpy_send_wqe. 128 */ 129 copysz = min_t(u64, *cur_edge - (void *)eseg->inline_hdr.start, 130 left); 131 memcpy(eseg->inline_hdr.start, pdata, copysz); 132 stride = ALIGN(sizeof(struct mlx5_wqe_eth_seg) - 133 sizeof(eseg->inline_hdr.start) + copysz, 16); 134 *size += stride / 16; 135 *seg += stride; 136 137 if (copysz < left) { 138 handle_post_send_edge(&qp->sq, seg, *size, cur_edge); 139 left -= copysz; 140 pdata += copysz; 141 memcpy_send_wqe(&qp->sq, cur_edge, seg, size, pdata, 142 left); 143 } 144 145 return; 146 } 147 148 *seg += sizeof(struct mlx5_wqe_eth_seg); 149 *size += sizeof(struct mlx5_wqe_eth_seg) / 16; 150 } 151 152 static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg, 153 const struct ib_send_wr *wr) 154 { 155 memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av)); 156 dseg->av.dqp_dct = 157 cpu_to_be32(ud_wr(wr)->remote_qpn | MLX5_EXTENDED_UD_AV); 158 dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey); 159 } 160 161 static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg) 162 { 163 dseg->byte_count = cpu_to_be32(sg->length); 164 dseg->lkey = cpu_to_be32(sg->lkey); 165 dseg->addr = cpu_to_be64(sg->addr); 166 } 167 168 static u64 get_xlt_octo(u64 bytes) 169 { 170 return ALIGN(bytes, MLX5_IB_UMR_XLT_ALIGNMENT) / 171 MLX5_IB_UMR_OCTOWORD; 172 } 173 174 static __be64 frwr_mkey_mask(bool atomic) 175 { 176 u64 result; 177 178 result = MLX5_MKEY_MASK_LEN | 179 MLX5_MKEY_MASK_PAGE_SIZE | 180 MLX5_MKEY_MASK_START_ADDR | 181 MLX5_MKEY_MASK_EN_RINVAL | 182 MLX5_MKEY_MASK_KEY | 183 MLX5_MKEY_MASK_LR | 184 MLX5_MKEY_MASK_LW | 185 MLX5_MKEY_MASK_RR | 186 MLX5_MKEY_MASK_RW | 187 MLX5_MKEY_MASK_SMALL_FENCE | 188 MLX5_MKEY_MASK_FREE; 189 190 if (atomic) 191 result |= MLX5_MKEY_MASK_A; 192 193 return cpu_to_be64(result); 194 } 195 196 static __be64 sig_mkey_mask(void) 197 { 198 u64 result; 199 200 result = MLX5_MKEY_MASK_LEN | 201 MLX5_MKEY_MASK_PAGE_SIZE | 202 MLX5_MKEY_MASK_START_ADDR | 203 MLX5_MKEY_MASK_EN_SIGERR | 204 MLX5_MKEY_MASK_EN_RINVAL | 205 MLX5_MKEY_MASK_KEY | 206 MLX5_MKEY_MASK_LR | 207 MLX5_MKEY_MASK_LW | 208 MLX5_MKEY_MASK_RR | 209 MLX5_MKEY_MASK_RW | 210 MLX5_MKEY_MASK_SMALL_FENCE | 211 MLX5_MKEY_MASK_FREE | 212 MLX5_MKEY_MASK_BSF_EN; 213 214 return cpu_to_be64(result); 215 } 216 217 static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr, 218 struct mlx5_ib_mr *mr, u8 flags, bool atomic) 219 { 220 int size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size; 221 222 memset(umr, 0, sizeof(*umr)); 223 224 umr->flags = flags; 225 umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size)); 226 umr->mkey_mask = frwr_mkey_mask(atomic); 227 } 228 229 static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr) 230 { 231 memset(umr, 0, sizeof(*umr)); 232 umr->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE); 233 umr->flags = MLX5_UMR_INLINE; 234 } 235 236 static __be64 get_umr_enable_mr_mask(void) 237 { 238 u64 result; 239 240 result = MLX5_MKEY_MASK_KEY | 241 MLX5_MKEY_MASK_FREE; 242 243 return cpu_to_be64(result); 244 } 245 246 static __be64 get_umr_disable_mr_mask(void) 247 { 248 u64 result; 249 250 result = MLX5_MKEY_MASK_FREE; 251 252 return cpu_to_be64(result); 253 } 254 255 static __be64 get_umr_update_translation_mask(void) 256 { 257 u64 result; 258 259 result = MLX5_MKEY_MASK_LEN | 260 MLX5_MKEY_MASK_PAGE_SIZE | 261 MLX5_MKEY_MASK_START_ADDR; 262 263 return cpu_to_be64(result); 264 } 265 266 static __be64 get_umr_update_access_mask(int atomic) 267 { 268 u64 result; 269 270 result = MLX5_MKEY_MASK_LR | 271 MLX5_MKEY_MASK_LW | 272 MLX5_MKEY_MASK_RR | 273 MLX5_MKEY_MASK_RW; 274 275 if (atomic) 276 result |= MLX5_MKEY_MASK_A; 277 278 return cpu_to_be64(result); 279 } 280 281 static __be64 get_umr_update_pd_mask(void) 282 { 283 u64 result; 284 285 result = MLX5_MKEY_MASK_PD; 286 287 return cpu_to_be64(result); 288 } 289 290 static int umr_check_mkey_mask(struct mlx5_ib_dev *dev, u64 mask) 291 { 292 if ((mask & MLX5_MKEY_MASK_PAGE_SIZE && 293 MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled)) || 294 (mask & MLX5_MKEY_MASK_A && 295 MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled))) 296 return -EPERM; 297 return 0; 298 } 299 300 static int set_reg_umr_segment(struct mlx5_ib_dev *dev, 301 struct mlx5_wqe_umr_ctrl_seg *umr, 302 const struct ib_send_wr *wr, int atomic) 303 { 304 const struct mlx5_umr_wr *umrwr = umr_wr(wr); 305 306 memset(umr, 0, sizeof(*umr)); 307 308 if (!umrwr->ignore_free_state) { 309 if (wr->send_flags & MLX5_IB_SEND_UMR_FAIL_IF_FREE) 310 /* fail if free */ 311 umr->flags = MLX5_UMR_CHECK_FREE; 312 else 313 /* fail if not free */ 314 umr->flags = MLX5_UMR_CHECK_NOT_FREE; 315 } 316 317 umr->xlt_octowords = cpu_to_be16(get_xlt_octo(umrwr->xlt_size)); 318 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_XLT) { 319 u64 offset = get_xlt_octo(umrwr->offset); 320 321 umr->xlt_offset = cpu_to_be16(offset & 0xffff); 322 umr->xlt_offset_47_16 = cpu_to_be32(offset >> 16); 323 umr->flags |= MLX5_UMR_TRANSLATION_OFFSET_EN; 324 } 325 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION) 326 umr->mkey_mask |= get_umr_update_translation_mask(); 327 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS) { 328 umr->mkey_mask |= get_umr_update_access_mask(atomic); 329 umr->mkey_mask |= get_umr_update_pd_mask(); 330 } 331 if (wr->send_flags & MLX5_IB_SEND_UMR_ENABLE_MR) 332 umr->mkey_mask |= get_umr_enable_mr_mask(); 333 if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR) 334 umr->mkey_mask |= get_umr_disable_mr_mask(); 335 336 if (!wr->num_sge) 337 umr->flags |= MLX5_UMR_INLINE; 338 339 return umr_check_mkey_mask(dev, be64_to_cpu(umr->mkey_mask)); 340 } 341 342 static u8 get_umr_flags(int acc) 343 { 344 return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC : 0) | 345 (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) | 346 (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) | 347 (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) | 348 MLX5_PERM_LOCAL_READ | MLX5_PERM_UMR_EN; 349 } 350 351 static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg, 352 struct mlx5_ib_mr *mr, 353 u32 key, int access) 354 { 355 int ndescs = ALIGN(mr->ndescs + mr->meta_ndescs, 8) >> 1; 356 357 memset(seg, 0, sizeof(*seg)); 358 359 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_MTT) 360 seg->log2_page_size = ilog2(mr->ibmr.page_size); 361 else if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS) 362 /* KLMs take twice the size of MTTs */ 363 ndescs *= 2; 364 365 seg->flags = get_umr_flags(access) | mr->access_mode; 366 seg->qpn_mkey7_0 = cpu_to_be32((key & 0xff) | 0xffffff00); 367 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL); 368 seg->start_addr = cpu_to_be64(mr->ibmr.iova); 369 seg->len = cpu_to_be64(mr->ibmr.length); 370 seg->xlt_oct_size = cpu_to_be32(ndescs); 371 } 372 373 static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg) 374 { 375 memset(seg, 0, sizeof(*seg)); 376 seg->status = MLX5_MKEY_STATUS_FREE; 377 } 378 379 static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg, 380 const struct ib_send_wr *wr) 381 { 382 const struct mlx5_umr_wr *umrwr = umr_wr(wr); 383 384 memset(seg, 0, sizeof(*seg)); 385 if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR) 386 seg->status = MLX5_MKEY_STATUS_FREE; 387 388 seg->flags = convert_access(umrwr->access_flags); 389 if (umrwr->pd) 390 seg->flags_pd = cpu_to_be32(to_mpd(umrwr->pd)->pdn); 391 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION && 392 !umrwr->length) 393 seg->flags_pd |= cpu_to_be32(MLX5_MKEY_LEN64); 394 395 seg->start_addr = cpu_to_be64(umrwr->virt_addr); 396 seg->len = cpu_to_be64(umrwr->length); 397 seg->log2_page_size = umrwr->page_shift; 398 seg->qpn_mkey7_0 = cpu_to_be32(0xffffff00 | 399 mlx5_mkey_variant(umrwr->mkey)); 400 } 401 402 static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg, 403 struct mlx5_ib_mr *mr, 404 struct mlx5_ib_pd *pd) 405 { 406 int bcount = mr->desc_size * (mr->ndescs + mr->meta_ndescs); 407 408 dseg->addr = cpu_to_be64(mr->desc_map); 409 dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64)); 410 dseg->lkey = cpu_to_be32(pd->ibpd.local_dma_lkey); 411 } 412 413 static __be32 send_ieth(const struct ib_send_wr *wr) 414 { 415 switch (wr->opcode) { 416 case IB_WR_SEND_WITH_IMM: 417 case IB_WR_RDMA_WRITE_WITH_IMM: 418 return wr->ex.imm_data; 419 420 case IB_WR_SEND_WITH_INV: 421 return cpu_to_be32(wr->ex.invalidate_rkey); 422 423 default: 424 return 0; 425 } 426 } 427 428 static u8 calc_sig(void *wqe, int size) 429 { 430 u8 *p = wqe; 431 u8 res = 0; 432 int i; 433 434 for (i = 0; i < size; i++) 435 res ^= p[i]; 436 437 return ~res; 438 } 439 440 static u8 wq_sig(void *wqe) 441 { 442 return calc_sig(wqe, (*((u8 *)wqe + 8) & 0x3f) << 4); 443 } 444 445 static int set_data_inl_seg(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr, 446 void **wqe, int *wqe_sz, void **cur_edge) 447 { 448 struct mlx5_wqe_inline_seg *seg; 449 size_t offset; 450 int inl = 0; 451 int i; 452 453 seg = *wqe; 454 *wqe += sizeof(*seg); 455 offset = sizeof(*seg); 456 457 for (i = 0; i < wr->num_sge; i++) { 458 size_t len = wr->sg_list[i].length; 459 void *addr = (void *)(unsigned long)(wr->sg_list[i].addr); 460 461 inl += len; 462 463 if (unlikely(inl > qp->max_inline_data)) 464 return -ENOMEM; 465 466 while (likely(len)) { 467 size_t leftlen; 468 size_t copysz; 469 470 handle_post_send_edge(&qp->sq, wqe, 471 *wqe_sz + (offset >> 4), 472 cur_edge); 473 474 leftlen = *cur_edge - *wqe; 475 copysz = min_t(size_t, leftlen, len); 476 477 memcpy(*wqe, addr, copysz); 478 len -= copysz; 479 addr += copysz; 480 *wqe += copysz; 481 offset += copysz; 482 } 483 } 484 485 seg->byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG); 486 487 *wqe_sz += ALIGN(inl + sizeof(seg->byte_count), 16) / 16; 488 489 return 0; 490 } 491 492 static u16 prot_field_size(enum ib_signature_type type) 493 { 494 switch (type) { 495 case IB_SIG_TYPE_T10_DIF: 496 return MLX5_DIF_SIZE; 497 default: 498 return 0; 499 } 500 } 501 502 static u8 bs_selector(int block_size) 503 { 504 switch (block_size) { 505 case 512: return 0x1; 506 case 520: return 0x2; 507 case 4096: return 0x3; 508 case 4160: return 0x4; 509 case 1073741824: return 0x5; 510 default: return 0; 511 } 512 } 513 514 static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain, 515 struct mlx5_bsf_inl *inl) 516 { 517 /* Valid inline section and allow BSF refresh */ 518 inl->vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID | 519 MLX5_BSF_REFRESH_DIF); 520 inl->dif_apptag = cpu_to_be16(domain->sig.dif.app_tag); 521 inl->dif_reftag = cpu_to_be32(domain->sig.dif.ref_tag); 522 /* repeating block */ 523 inl->rp_inv_seed = MLX5_BSF_REPEAT_BLOCK; 524 inl->sig_type = domain->sig.dif.bg_type == IB_T10DIF_CRC ? 525 MLX5_DIF_CRC : MLX5_DIF_IPCS; 526 527 if (domain->sig.dif.ref_remap) 528 inl->dif_inc_ref_guard_check |= MLX5_BSF_INC_REFTAG; 529 530 if (domain->sig.dif.app_escape) { 531 if (domain->sig.dif.ref_escape) 532 inl->dif_inc_ref_guard_check |= MLX5_BSF_APPREF_ESCAPE; 533 else 534 inl->dif_inc_ref_guard_check |= MLX5_BSF_APPTAG_ESCAPE; 535 } 536 537 inl->dif_app_bitmask_check = 538 cpu_to_be16(domain->sig.dif.apptag_check_mask); 539 } 540 541 static int mlx5_set_bsf(struct ib_mr *sig_mr, 542 struct ib_sig_attrs *sig_attrs, 543 struct mlx5_bsf *bsf, u32 data_size) 544 { 545 struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)->sig; 546 struct mlx5_bsf_basic *basic = &bsf->basic; 547 struct ib_sig_domain *mem = &sig_attrs->mem; 548 struct ib_sig_domain *wire = &sig_attrs->wire; 549 550 memset(bsf, 0, sizeof(*bsf)); 551 552 /* Basic + Extended + Inline */ 553 basic->bsf_size_sbs = 1 << 7; 554 /* Input domain check byte mask */ 555 basic->check_byte_mask = sig_attrs->check_mask; 556 basic->raw_data_size = cpu_to_be32(data_size); 557 558 /* Memory domain */ 559 switch (sig_attrs->mem.sig_type) { 560 case IB_SIG_TYPE_NONE: 561 break; 562 case IB_SIG_TYPE_T10_DIF: 563 basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval); 564 basic->m_bfs_psv = cpu_to_be32(msig->psv_memory.psv_idx); 565 mlx5_fill_inl_bsf(mem, &bsf->m_inl); 566 break; 567 default: 568 return -EINVAL; 569 } 570 571 /* Wire domain */ 572 switch (sig_attrs->wire.sig_type) { 573 case IB_SIG_TYPE_NONE: 574 break; 575 case IB_SIG_TYPE_T10_DIF: 576 if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval && 577 mem->sig_type == wire->sig_type) { 578 /* Same block structure */ 579 basic->bsf_size_sbs |= 1 << 4; 580 if (mem->sig.dif.bg_type == wire->sig.dif.bg_type) 581 basic->wire.copy_byte_mask |= MLX5_CPY_GRD_MASK; 582 if (mem->sig.dif.app_tag == wire->sig.dif.app_tag) 583 basic->wire.copy_byte_mask |= MLX5_CPY_APP_MASK; 584 if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag) 585 basic->wire.copy_byte_mask |= MLX5_CPY_REF_MASK; 586 } else 587 basic->wire.bs_selector = 588 bs_selector(wire->sig.dif.pi_interval); 589 590 basic->w_bfs_psv = cpu_to_be32(msig->psv_wire.psv_idx); 591 mlx5_fill_inl_bsf(wire, &bsf->w_inl); 592 break; 593 default: 594 return -EINVAL; 595 } 596 597 return 0; 598 } 599 600 601 static int set_sig_data_segment(const struct ib_send_wr *send_wr, 602 struct ib_mr *sig_mr, 603 struct ib_sig_attrs *sig_attrs, 604 struct mlx5_ib_qp *qp, void **seg, int *size, 605 void **cur_edge) 606 { 607 struct mlx5_bsf *bsf; 608 u32 data_len; 609 u32 data_key; 610 u64 data_va; 611 u32 prot_len = 0; 612 u32 prot_key = 0; 613 u64 prot_va = 0; 614 bool prot = false; 615 int ret; 616 int wqe_size; 617 struct mlx5_ib_mr *mr = to_mmr(sig_mr); 618 struct mlx5_ib_mr *pi_mr = mr->pi_mr; 619 620 data_len = pi_mr->data_length; 621 data_key = pi_mr->ibmr.lkey; 622 data_va = pi_mr->data_iova; 623 if (pi_mr->meta_ndescs) { 624 prot_len = pi_mr->meta_length; 625 prot_key = pi_mr->ibmr.lkey; 626 prot_va = pi_mr->pi_iova; 627 prot = true; 628 } 629 630 if (!prot || (data_key == prot_key && data_va == prot_va && 631 data_len == prot_len)) { 632 /** 633 * Source domain doesn't contain signature information 634 * or data and protection are interleaved in memory. 635 * So need construct: 636 * ------------------ 637 * | data_klm | 638 * ------------------ 639 * | BSF | 640 * ------------------ 641 **/ 642 struct mlx5_klm *data_klm = *seg; 643 644 data_klm->bcount = cpu_to_be32(data_len); 645 data_klm->key = cpu_to_be32(data_key); 646 data_klm->va = cpu_to_be64(data_va); 647 wqe_size = ALIGN(sizeof(*data_klm), 64); 648 } else { 649 /** 650 * Source domain contains signature information 651 * So need construct a strided block format: 652 * --------------------------- 653 * | stride_block_ctrl | 654 * --------------------------- 655 * | data_klm | 656 * --------------------------- 657 * | prot_klm | 658 * --------------------------- 659 * | BSF | 660 * --------------------------- 661 **/ 662 struct mlx5_stride_block_ctrl_seg *sblock_ctrl; 663 struct mlx5_stride_block_entry *data_sentry; 664 struct mlx5_stride_block_entry *prot_sentry; 665 u16 block_size = sig_attrs->mem.sig.dif.pi_interval; 666 int prot_size; 667 668 sblock_ctrl = *seg; 669 data_sentry = (void *)sblock_ctrl + sizeof(*sblock_ctrl); 670 prot_sentry = (void *)data_sentry + sizeof(*data_sentry); 671 672 prot_size = prot_field_size(sig_attrs->mem.sig_type); 673 if (!prot_size) { 674 pr_err("Bad block size given: %u\n", block_size); 675 return -EINVAL; 676 } 677 sblock_ctrl->bcount_per_cycle = cpu_to_be32(block_size + 678 prot_size); 679 sblock_ctrl->op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP); 680 sblock_ctrl->repeat_count = cpu_to_be32(data_len / block_size); 681 sblock_ctrl->num_entries = cpu_to_be16(2); 682 683 data_sentry->bcount = cpu_to_be16(block_size); 684 data_sentry->key = cpu_to_be32(data_key); 685 data_sentry->va = cpu_to_be64(data_va); 686 data_sentry->stride = cpu_to_be16(block_size); 687 688 prot_sentry->bcount = cpu_to_be16(prot_size); 689 prot_sentry->key = cpu_to_be32(prot_key); 690 prot_sentry->va = cpu_to_be64(prot_va); 691 prot_sentry->stride = cpu_to_be16(prot_size); 692 693 wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) + 694 sizeof(*prot_sentry), 64); 695 } 696 697 *seg += wqe_size; 698 *size += wqe_size / 16; 699 handle_post_send_edge(&qp->sq, seg, *size, cur_edge); 700 701 bsf = *seg; 702 ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len); 703 if (ret) 704 return -EINVAL; 705 706 *seg += sizeof(*bsf); 707 *size += sizeof(*bsf) / 16; 708 handle_post_send_edge(&qp->sq, seg, *size, cur_edge); 709 710 return 0; 711 } 712 713 static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg, 714 struct ib_mr *sig_mr, int access_flags, 715 u32 size, u32 length, u32 pdn) 716 { 717 u32 sig_key = sig_mr->rkey; 718 u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1; 719 720 memset(seg, 0, sizeof(*seg)); 721 722 seg->flags = get_umr_flags(access_flags) | MLX5_MKC_ACCESS_MODE_KLMS; 723 seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00); 724 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 | 725 MLX5_MKEY_BSF_EN | pdn); 726 seg->len = cpu_to_be64(length); 727 seg->xlt_oct_size = cpu_to_be32(get_xlt_octo(size)); 728 seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE); 729 } 730 731 static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr, 732 u32 size) 733 { 734 memset(umr, 0, sizeof(*umr)); 735 736 umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE; 737 umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size)); 738 umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE); 739 umr->mkey_mask = sig_mkey_mask(); 740 } 741 742 static int set_pi_umr_wr(const struct ib_send_wr *send_wr, 743 struct mlx5_ib_qp *qp, void **seg, int *size, 744 void **cur_edge) 745 { 746 const struct ib_reg_wr *wr = reg_wr(send_wr); 747 struct mlx5_ib_mr *sig_mr = to_mmr(wr->mr); 748 struct mlx5_ib_mr *pi_mr = sig_mr->pi_mr; 749 struct ib_sig_attrs *sig_attrs = sig_mr->ibmr.sig_attrs; 750 u32 pdn = to_mpd(qp->ibqp.pd)->pdn; 751 u32 xlt_size; 752 int region_len, ret; 753 754 if (unlikely(send_wr->num_sge != 0) || 755 unlikely(wr->access & IB_ACCESS_REMOTE_ATOMIC) || 756 unlikely(!sig_mr->sig) || unlikely(!qp->ibqp.integrity_en) || 757 unlikely(!sig_mr->sig->sig_status_checked)) 758 return -EINVAL; 759 760 /* length of the protected region, data + protection */ 761 region_len = pi_mr->ibmr.length; 762 763 /** 764 * KLM octoword size - if protection was provided 765 * then we use strided block format (3 octowords), 766 * else we use single KLM (1 octoword) 767 **/ 768 if (sig_attrs->mem.sig_type != IB_SIG_TYPE_NONE) 769 xlt_size = 0x30; 770 else 771 xlt_size = sizeof(struct mlx5_klm); 772 773 set_sig_umr_segment(*seg, xlt_size); 774 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg); 775 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; 776 handle_post_send_edge(&qp->sq, seg, *size, cur_edge); 777 778 set_sig_mkey_segment(*seg, wr->mr, wr->access, xlt_size, region_len, 779 pdn); 780 *seg += sizeof(struct mlx5_mkey_seg); 781 *size += sizeof(struct mlx5_mkey_seg) / 16; 782 handle_post_send_edge(&qp->sq, seg, *size, cur_edge); 783 784 ret = set_sig_data_segment(send_wr, wr->mr, sig_attrs, qp, seg, size, 785 cur_edge); 786 if (ret) 787 return ret; 788 789 sig_mr->sig->sig_status_checked = false; 790 return 0; 791 } 792 793 static int set_psv_wr(struct ib_sig_domain *domain, 794 u32 psv_idx, void **seg, int *size) 795 { 796 struct mlx5_seg_set_psv *psv_seg = *seg; 797 798 memset(psv_seg, 0, sizeof(*psv_seg)); 799 psv_seg->psv_num = cpu_to_be32(psv_idx); 800 switch (domain->sig_type) { 801 case IB_SIG_TYPE_NONE: 802 break; 803 case IB_SIG_TYPE_T10_DIF: 804 psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 | 805 domain->sig.dif.app_tag); 806 psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag); 807 break; 808 default: 809 pr_err("Bad signature type (%d) is given.\n", 810 domain->sig_type); 811 return -EINVAL; 812 } 813 814 *seg += sizeof(*psv_seg); 815 *size += sizeof(*psv_seg) / 16; 816 817 return 0; 818 } 819 820 static int set_reg_wr(struct mlx5_ib_qp *qp, 821 const struct ib_reg_wr *wr, 822 void **seg, int *size, void **cur_edge, 823 bool check_not_free) 824 { 825 struct mlx5_ib_mr *mr = to_mmr(wr->mr); 826 struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd); 827 struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device); 828 int mr_list_size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size; 829 bool umr_inline = mr_list_size <= MLX5_IB_SQ_UMR_INLINE_THRESHOLD; 830 bool atomic = wr->access & IB_ACCESS_REMOTE_ATOMIC; 831 u8 flags = 0; 832 833 if (!mlx5_ib_can_use_umr(dev, atomic, wr->access)) { 834 mlx5_ib_warn(to_mdev(qp->ibqp.device), 835 "Fast update of %s for MR is disabled\n", 836 (MLX5_CAP_GEN(dev->mdev, 837 umr_modify_entity_size_disabled)) ? 838 "entity size" : 839 "atomic access"); 840 return -EINVAL; 841 } 842 843 if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) { 844 mlx5_ib_warn(to_mdev(qp->ibqp.device), 845 "Invalid IB_SEND_INLINE send flag\n"); 846 return -EINVAL; 847 } 848 849 if (check_not_free) 850 flags |= MLX5_UMR_CHECK_NOT_FREE; 851 if (umr_inline) 852 flags |= MLX5_UMR_INLINE; 853 854 set_reg_umr_seg(*seg, mr, flags, atomic); 855 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg); 856 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; 857 handle_post_send_edge(&qp->sq, seg, *size, cur_edge); 858 859 set_reg_mkey_seg(*seg, mr, wr->key, wr->access); 860 *seg += sizeof(struct mlx5_mkey_seg); 861 *size += sizeof(struct mlx5_mkey_seg) / 16; 862 handle_post_send_edge(&qp->sq, seg, *size, cur_edge); 863 864 if (umr_inline) { 865 memcpy_send_wqe(&qp->sq, cur_edge, seg, size, mr->descs, 866 mr_list_size); 867 *size = ALIGN(*size, MLX5_SEND_WQE_BB >> 4); 868 } else { 869 set_reg_data_seg(*seg, mr, pd); 870 *seg += sizeof(struct mlx5_wqe_data_seg); 871 *size += (sizeof(struct mlx5_wqe_data_seg) / 16); 872 } 873 return 0; 874 } 875 876 static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size, 877 void **cur_edge) 878 { 879 set_linv_umr_seg(*seg); 880 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg); 881 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; 882 handle_post_send_edge(&qp->sq, seg, *size, cur_edge); 883 set_linv_mkey_seg(*seg); 884 *seg += sizeof(struct mlx5_mkey_seg); 885 *size += sizeof(struct mlx5_mkey_seg) / 16; 886 handle_post_send_edge(&qp->sq, seg, *size, cur_edge); 887 } 888 889 static void dump_wqe(struct mlx5_ib_qp *qp, u32 idx, int size_16) 890 { 891 __be32 *p = NULL; 892 int i, j; 893 894 pr_debug("dump WQE index %u:\n", idx); 895 for (i = 0, j = 0; i < size_16 * 4; i += 4, j += 4) { 896 if ((i & 0xf) == 0) { 897 p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, idx); 898 pr_debug("WQBB at %p:\n", (void *)p); 899 j = 0; 900 idx = (idx + 1) & (qp->sq.wqe_cnt - 1); 901 } 902 pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p[j]), 903 be32_to_cpu(p[j + 1]), be32_to_cpu(p[j + 2]), 904 be32_to_cpu(p[j + 3])); 905 } 906 } 907 908 static int __begin_wqe(struct mlx5_ib_qp *qp, void **seg, 909 struct mlx5_wqe_ctrl_seg **ctrl, 910 const struct ib_send_wr *wr, unsigned int *idx, 911 int *size, void **cur_edge, int nreq, 912 bool send_signaled, bool solicited) 913 { 914 if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq))) 915 return -ENOMEM; 916 917 *idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1); 918 *seg = mlx5_frag_buf_get_wqe(&qp->sq.fbc, *idx); 919 *ctrl = *seg; 920 *(uint32_t *)(*seg + 8) = 0; 921 (*ctrl)->imm = send_ieth(wr); 922 (*ctrl)->fm_ce_se = qp->sq_signal_bits | 923 (send_signaled ? MLX5_WQE_CTRL_CQ_UPDATE : 0) | 924 (solicited ? MLX5_WQE_CTRL_SOLICITED : 0); 925 926 *seg += sizeof(**ctrl); 927 *size = sizeof(**ctrl) / 16; 928 *cur_edge = qp->sq.cur_edge; 929 930 return 0; 931 } 932 933 static int begin_wqe(struct mlx5_ib_qp *qp, void **seg, 934 struct mlx5_wqe_ctrl_seg **ctrl, 935 const struct ib_send_wr *wr, unsigned int *idx, int *size, 936 void **cur_edge, int nreq) 937 { 938 return __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq, 939 wr->send_flags & IB_SEND_SIGNALED, 940 wr->send_flags & IB_SEND_SOLICITED); 941 } 942 943 static void finish_wqe(struct mlx5_ib_qp *qp, 944 struct mlx5_wqe_ctrl_seg *ctrl, 945 void *seg, u8 size, void *cur_edge, 946 unsigned int idx, u64 wr_id, int nreq, u8 fence, 947 u32 mlx5_opcode) 948 { 949 u8 opmod = 0; 950 951 ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) | 952 mlx5_opcode | ((u32)opmod << 24)); 953 ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8)); 954 ctrl->fm_ce_se |= fence; 955 if (unlikely(qp->flags_en & MLX5_QP_FLAG_SIGNATURE)) 956 ctrl->signature = wq_sig(ctrl); 957 958 qp->sq.wrid[idx] = wr_id; 959 qp->sq.w_list[idx].opcode = mlx5_opcode; 960 qp->sq.wqe_head[idx] = qp->sq.head + nreq; 961 qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB); 962 qp->sq.w_list[idx].next = qp->sq.cur_post; 963 964 /* We save the edge which was possibly updated during the WQE 965 * construction, into SQ's cache. 966 */ 967 seg = PTR_ALIGN(seg, MLX5_SEND_WQE_BB); 968 qp->sq.cur_edge = (unlikely(seg == cur_edge)) ? 969 get_sq_edge(&qp->sq, qp->sq.cur_post & 970 (qp->sq.wqe_cnt - 1)) : 971 cur_edge; 972 } 973 974 static void handle_rdma_op(const struct ib_send_wr *wr, void **seg, int *size) 975 { 976 set_raddr_seg(*seg, rdma_wr(wr)->remote_addr, rdma_wr(wr)->rkey); 977 *seg += sizeof(struct mlx5_wqe_raddr_seg); 978 *size += sizeof(struct mlx5_wqe_raddr_seg) / 16; 979 } 980 981 static void handle_local_inv(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr, 982 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, 983 int *size, void **cur_edge, unsigned int idx) 984 { 985 qp->sq.wr_data[idx] = IB_WR_LOCAL_INV; 986 (*ctrl)->imm = cpu_to_be32(wr->ex.invalidate_rkey); 987 set_linv_wr(qp, seg, size, cur_edge); 988 } 989 990 static int handle_reg_mr(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr, 991 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size, 992 void **cur_edge, unsigned int idx) 993 { 994 qp->sq.wr_data[idx] = IB_WR_REG_MR; 995 (*ctrl)->imm = cpu_to_be32(reg_wr(wr)->key); 996 return set_reg_wr(qp, reg_wr(wr), seg, size, cur_edge, true); 997 } 998 999 static int handle_psv(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, 1000 const struct ib_send_wr *wr, 1001 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size, 1002 void **cur_edge, unsigned int *idx, int nreq, 1003 struct ib_sig_domain *domain, u32 psv_index, 1004 u8 next_fence) 1005 { 1006 int err; 1007 1008 /* 1009 * SET_PSV WQEs are not signaled and solicited on error. 1010 */ 1011 err = __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq, 1012 false, true); 1013 if (unlikely(err)) { 1014 mlx5_ib_warn(dev, "\n"); 1015 err = -ENOMEM; 1016 goto out; 1017 } 1018 err = set_psv_wr(domain, psv_index, seg, size); 1019 if (unlikely(err)) { 1020 mlx5_ib_warn(dev, "\n"); 1021 goto out; 1022 } 1023 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, nreq, 1024 next_fence, MLX5_OPCODE_SET_PSV); 1025 1026 out: 1027 return err; 1028 } 1029 1030 static int handle_reg_mr_integrity(struct mlx5_ib_dev *dev, 1031 struct mlx5_ib_qp *qp, 1032 const struct ib_send_wr *wr, 1033 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, 1034 int *size, void **cur_edge, 1035 unsigned int *idx, int nreq, u8 fence, 1036 u8 next_fence) 1037 { 1038 struct mlx5_ib_mr *mr; 1039 struct mlx5_ib_mr *pi_mr; 1040 struct mlx5_ib_mr pa_pi_mr; 1041 struct ib_sig_attrs *sig_attrs; 1042 struct ib_reg_wr reg_pi_wr; 1043 int err; 1044 1045 qp->sq.wr_data[*idx] = IB_WR_REG_MR_INTEGRITY; 1046 1047 mr = to_mmr(reg_wr(wr)->mr); 1048 pi_mr = mr->pi_mr; 1049 1050 if (pi_mr) { 1051 memset(®_pi_wr, 0, 1052 sizeof(struct ib_reg_wr)); 1053 1054 reg_pi_wr.mr = &pi_mr->ibmr; 1055 reg_pi_wr.access = reg_wr(wr)->access; 1056 reg_pi_wr.key = pi_mr->ibmr.rkey; 1057 1058 (*ctrl)->imm = cpu_to_be32(reg_pi_wr.key); 1059 /* UMR for data + prot registration */ 1060 err = set_reg_wr(qp, ®_pi_wr, seg, size, cur_edge, false); 1061 if (unlikely(err)) 1062 goto out; 1063 1064 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, 1065 nreq, fence, MLX5_OPCODE_UMR); 1066 1067 err = begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq); 1068 if (unlikely(err)) { 1069 mlx5_ib_warn(dev, "\n"); 1070 err = -ENOMEM; 1071 goto out; 1072 } 1073 } else { 1074 memset(&pa_pi_mr, 0, sizeof(struct mlx5_ib_mr)); 1075 /* No UMR, use local_dma_lkey */ 1076 pa_pi_mr.ibmr.lkey = mr->ibmr.pd->local_dma_lkey; 1077 pa_pi_mr.ndescs = mr->ndescs; 1078 pa_pi_mr.data_length = mr->data_length; 1079 pa_pi_mr.data_iova = mr->data_iova; 1080 if (mr->meta_ndescs) { 1081 pa_pi_mr.meta_ndescs = mr->meta_ndescs; 1082 pa_pi_mr.meta_length = mr->meta_length; 1083 pa_pi_mr.pi_iova = mr->pi_iova; 1084 } 1085 1086 pa_pi_mr.ibmr.length = mr->ibmr.length; 1087 mr->pi_mr = &pa_pi_mr; 1088 } 1089 (*ctrl)->imm = cpu_to_be32(mr->ibmr.rkey); 1090 /* UMR for sig MR */ 1091 err = set_pi_umr_wr(wr, qp, seg, size, cur_edge); 1092 if (unlikely(err)) { 1093 mlx5_ib_warn(dev, "\n"); 1094 goto out; 1095 } 1096 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, nreq, 1097 fence, MLX5_OPCODE_UMR); 1098 1099 sig_attrs = mr->ibmr.sig_attrs; 1100 err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq, 1101 &sig_attrs->mem, mr->sig->psv_memory.psv_idx, 1102 next_fence); 1103 if (unlikely(err)) 1104 goto out; 1105 1106 err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq, 1107 &sig_attrs->wire, mr->sig->psv_wire.psv_idx, 1108 next_fence); 1109 if (unlikely(err)) 1110 goto out; 1111 1112 qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL; 1113 1114 out: 1115 return err; 1116 } 1117 1118 static int handle_qpt_rc(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, 1119 const struct ib_send_wr *wr, 1120 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size, 1121 void **cur_edge, unsigned int *idx, int nreq, u8 fence, 1122 u8 next_fence, int *num_sge) 1123 { 1124 int err = 0; 1125 1126 switch (wr->opcode) { 1127 case IB_WR_RDMA_READ: 1128 case IB_WR_RDMA_WRITE: 1129 case IB_WR_RDMA_WRITE_WITH_IMM: 1130 handle_rdma_op(wr, seg, size); 1131 break; 1132 1133 case IB_WR_ATOMIC_CMP_AND_SWP: 1134 case IB_WR_ATOMIC_FETCH_AND_ADD: 1135 case IB_WR_MASKED_ATOMIC_CMP_AND_SWP: 1136 mlx5_ib_warn(dev, "Atomic operations are not supported yet\n"); 1137 err = -EOPNOTSUPP; 1138 goto out; 1139 1140 case IB_WR_LOCAL_INV: 1141 handle_local_inv(qp, wr, ctrl, seg, size, cur_edge, *idx); 1142 *num_sge = 0; 1143 break; 1144 1145 case IB_WR_REG_MR: 1146 err = handle_reg_mr(qp, wr, ctrl, seg, size, cur_edge, *idx); 1147 if (unlikely(err)) 1148 goto out; 1149 *num_sge = 0; 1150 break; 1151 1152 case IB_WR_REG_MR_INTEGRITY: 1153 err = handle_reg_mr_integrity(dev, qp, wr, ctrl, seg, size, 1154 cur_edge, idx, nreq, fence, 1155 next_fence); 1156 if (unlikely(err)) 1157 goto out; 1158 *num_sge = 0; 1159 break; 1160 1161 default: 1162 break; 1163 } 1164 1165 out: 1166 return err; 1167 } 1168 1169 static void handle_qpt_uc(const struct ib_send_wr *wr, void **seg, int *size) 1170 { 1171 switch (wr->opcode) { 1172 case IB_WR_RDMA_WRITE: 1173 case IB_WR_RDMA_WRITE_WITH_IMM: 1174 handle_rdma_op(wr, seg, size); 1175 break; 1176 default: 1177 break; 1178 } 1179 } 1180 1181 static void handle_qpt_hw_gsi(struct mlx5_ib_qp *qp, 1182 const struct ib_send_wr *wr, void **seg, 1183 int *size, void **cur_edge) 1184 { 1185 set_datagram_seg(*seg, wr); 1186 *seg += sizeof(struct mlx5_wqe_datagram_seg); 1187 *size += sizeof(struct mlx5_wqe_datagram_seg) / 16; 1188 handle_post_send_edge(&qp->sq, seg, *size, cur_edge); 1189 } 1190 1191 static void handle_qpt_ud(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr, 1192 void **seg, int *size, void **cur_edge) 1193 { 1194 set_datagram_seg(*seg, wr); 1195 *seg += sizeof(struct mlx5_wqe_datagram_seg); 1196 *size += sizeof(struct mlx5_wqe_datagram_seg) / 16; 1197 handle_post_send_edge(&qp->sq, seg, *size, cur_edge); 1198 1199 /* handle qp that supports ud offload */ 1200 if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO) { 1201 struct mlx5_wqe_eth_pad *pad; 1202 1203 pad = *seg; 1204 memset(pad, 0, sizeof(struct mlx5_wqe_eth_pad)); 1205 *seg += sizeof(struct mlx5_wqe_eth_pad); 1206 *size += sizeof(struct mlx5_wqe_eth_pad) / 16; 1207 set_eth_seg(wr, qp, seg, size, cur_edge); 1208 handle_post_send_edge(&qp->sq, seg, *size, cur_edge); 1209 } 1210 } 1211 1212 static int handle_qpt_reg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, 1213 const struct ib_send_wr *wr, 1214 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, 1215 int *size, void **cur_edge, unsigned int idx) 1216 { 1217 int err = 0; 1218 1219 if (unlikely(wr->opcode != MLX5_IB_WR_UMR)) { 1220 err = -EINVAL; 1221 mlx5_ib_warn(dev, "bad opcode %d\n", wr->opcode); 1222 goto out; 1223 } 1224 1225 qp->sq.wr_data[idx] = MLX5_IB_WR_UMR; 1226 (*ctrl)->imm = cpu_to_be32(umr_wr(wr)->mkey); 1227 err = set_reg_umr_segment(dev, *seg, wr, 1228 !!(MLX5_CAP_GEN(dev->mdev, atomic))); 1229 if (unlikely(err)) 1230 goto out; 1231 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg); 1232 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; 1233 handle_post_send_edge(&qp->sq, seg, *size, cur_edge); 1234 set_reg_mkey_segment(*seg, wr); 1235 *seg += sizeof(struct mlx5_mkey_seg); 1236 *size += sizeof(struct mlx5_mkey_seg) / 16; 1237 handle_post_send_edge(&qp->sq, seg, *size, cur_edge); 1238 out: 1239 return err; 1240 } 1241 1242 int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 1243 const struct ib_send_wr **bad_wr, bool drain) 1244 { 1245 struct mlx5_wqe_ctrl_seg *ctrl = NULL; /* compiler warning */ 1246 struct mlx5_ib_dev *dev = to_mdev(ibqp->device); 1247 struct mlx5_core_dev *mdev = dev->mdev; 1248 struct mlx5_ib_qp *qp; 1249 struct mlx5_wqe_xrc_seg *xrc; 1250 struct mlx5_bf *bf; 1251 void *cur_edge; 1252 int uninitialized_var(size); 1253 unsigned long flags; 1254 unsigned int idx; 1255 int err = 0; 1256 int num_sge; 1257 void *seg; 1258 int nreq; 1259 int i; 1260 u8 next_fence = 0; 1261 u8 fence; 1262 1263 if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR && 1264 !drain)) { 1265 *bad_wr = wr; 1266 return -EIO; 1267 } 1268 1269 if (unlikely(ibqp->qp_type == IB_QPT_GSI)) 1270 return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr); 1271 1272 qp = to_mqp(ibqp); 1273 bf = &qp->bf; 1274 1275 spin_lock_irqsave(&qp->sq.lock, flags); 1276 1277 for (nreq = 0; wr; nreq++, wr = wr->next) { 1278 if (unlikely(wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) { 1279 mlx5_ib_warn(dev, "\n"); 1280 err = -EINVAL; 1281 *bad_wr = wr; 1282 goto out; 1283 } 1284 1285 num_sge = wr->num_sge; 1286 if (unlikely(num_sge > qp->sq.max_gs)) { 1287 mlx5_ib_warn(dev, "\n"); 1288 err = -EINVAL; 1289 *bad_wr = wr; 1290 goto out; 1291 } 1292 1293 err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, &cur_edge, 1294 nreq); 1295 if (err) { 1296 mlx5_ib_warn(dev, "\n"); 1297 err = -ENOMEM; 1298 *bad_wr = wr; 1299 goto out; 1300 } 1301 1302 if (wr->opcode == IB_WR_REG_MR || 1303 wr->opcode == IB_WR_REG_MR_INTEGRITY) { 1304 fence = dev->umr_fence; 1305 next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL; 1306 } else { 1307 if (wr->send_flags & IB_SEND_FENCE) { 1308 if (qp->next_fence) 1309 fence = MLX5_FENCE_MODE_SMALL_AND_FENCE; 1310 else 1311 fence = MLX5_FENCE_MODE_FENCE; 1312 } else { 1313 fence = qp->next_fence; 1314 } 1315 } 1316 1317 switch (ibqp->qp_type) { 1318 case IB_QPT_XRC_INI: 1319 xrc = seg; 1320 seg += sizeof(*xrc); 1321 size += sizeof(*xrc) / 16; 1322 fallthrough; 1323 case IB_QPT_RC: 1324 err = handle_qpt_rc(dev, qp, wr, &ctrl, &seg, &size, 1325 &cur_edge, &idx, nreq, fence, 1326 next_fence, &num_sge); 1327 if (unlikely(err)) { 1328 *bad_wr = wr; 1329 goto out; 1330 } else if (wr->opcode == IB_WR_REG_MR_INTEGRITY) { 1331 goto skip_psv; 1332 } 1333 break; 1334 1335 case IB_QPT_UC: 1336 handle_qpt_uc(wr, &seg, &size); 1337 break; 1338 case IB_QPT_SMI: 1339 if (unlikely(!mdev->port_caps[qp->port - 1].has_smi)) { 1340 mlx5_ib_warn(dev, "Send SMP MADs is not allowed\n"); 1341 err = -EPERM; 1342 *bad_wr = wr; 1343 goto out; 1344 } 1345 fallthrough; 1346 case MLX5_IB_QPT_HW_GSI: 1347 handle_qpt_hw_gsi(qp, wr, &seg, &size, &cur_edge); 1348 break; 1349 case IB_QPT_UD: 1350 handle_qpt_ud(qp, wr, &seg, &size, &cur_edge); 1351 break; 1352 case MLX5_IB_QPT_REG_UMR: 1353 err = handle_qpt_reg_umr(dev, qp, wr, &ctrl, &seg, 1354 &size, &cur_edge, idx); 1355 if (unlikely(err)) 1356 goto out; 1357 break; 1358 1359 default: 1360 break; 1361 } 1362 1363 if (wr->send_flags & IB_SEND_INLINE && num_sge) { 1364 err = set_data_inl_seg(qp, wr, &seg, &size, &cur_edge); 1365 if (unlikely(err)) { 1366 mlx5_ib_warn(dev, "\n"); 1367 *bad_wr = wr; 1368 goto out; 1369 } 1370 } else { 1371 for (i = 0; i < num_sge; i++) { 1372 handle_post_send_edge(&qp->sq, &seg, size, 1373 &cur_edge); 1374 if (unlikely(!wr->sg_list[i].length)) 1375 continue; 1376 1377 set_data_ptr_seg( 1378 (struct mlx5_wqe_data_seg *)seg, 1379 wr->sg_list + i); 1380 size += sizeof(struct mlx5_wqe_data_seg) / 16; 1381 seg += sizeof(struct mlx5_wqe_data_seg); 1382 } 1383 } 1384 1385 qp->next_fence = next_fence; 1386 finish_wqe(qp, ctrl, seg, size, cur_edge, idx, wr->wr_id, nreq, 1387 fence, mlx5_ib_opcode[wr->opcode]); 1388 skip_psv: 1389 if (0) 1390 dump_wqe(qp, idx, size); 1391 } 1392 1393 out: 1394 if (likely(nreq)) { 1395 qp->sq.head += nreq; 1396 1397 /* Make sure that descriptors are written before 1398 * updating doorbell record and ringing the doorbell 1399 */ 1400 wmb(); 1401 1402 qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post); 1403 1404 /* Make sure doorbell record is visible to the HCA before 1405 * we hit doorbell. 1406 */ 1407 wmb(); 1408 1409 mlx5_write64((__be32 *)ctrl, bf->bfreg->map + bf->offset); 1410 /* Make sure doorbells don't leak out of SQ spinlock 1411 * and reach the HCA out of order. 1412 */ 1413 bf->offset ^= bf->buf_size; 1414 } 1415 1416 spin_unlock_irqrestore(&qp->sq.lock, flags); 1417 1418 return err; 1419 } 1420 1421 static void set_sig_seg(struct mlx5_rwqe_sig *sig, int max_gs) 1422 { 1423 sig->signature = calc_sig(sig, (max_gs + 1) << 2); 1424 } 1425 1426 int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 1427 const struct ib_recv_wr **bad_wr, bool drain) 1428 { 1429 struct mlx5_ib_qp *qp = to_mqp(ibqp); 1430 struct mlx5_wqe_data_seg *scat; 1431 struct mlx5_rwqe_sig *sig; 1432 struct mlx5_ib_dev *dev = to_mdev(ibqp->device); 1433 struct mlx5_core_dev *mdev = dev->mdev; 1434 unsigned long flags; 1435 int err = 0; 1436 int nreq; 1437 int ind; 1438 int i; 1439 1440 if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR && 1441 !drain)) { 1442 *bad_wr = wr; 1443 return -EIO; 1444 } 1445 1446 if (unlikely(ibqp->qp_type == IB_QPT_GSI)) 1447 return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr); 1448 1449 spin_lock_irqsave(&qp->rq.lock, flags); 1450 1451 ind = qp->rq.head & (qp->rq.wqe_cnt - 1); 1452 1453 for (nreq = 0; wr; nreq++, wr = wr->next) { 1454 if (mlx5_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) { 1455 err = -ENOMEM; 1456 *bad_wr = wr; 1457 goto out; 1458 } 1459 1460 if (unlikely(wr->num_sge > qp->rq.max_gs)) { 1461 err = -EINVAL; 1462 *bad_wr = wr; 1463 goto out; 1464 } 1465 1466 scat = mlx5_frag_buf_get_wqe(&qp->rq.fbc, ind); 1467 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE) 1468 scat++; 1469 1470 for (i = 0; i < wr->num_sge; i++) 1471 set_data_ptr_seg(scat + i, wr->sg_list + i); 1472 1473 if (i < qp->rq.max_gs) { 1474 scat[i].byte_count = 0; 1475 scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY); 1476 scat[i].addr = 0; 1477 } 1478 1479 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE) { 1480 sig = (struct mlx5_rwqe_sig *)scat; 1481 set_sig_seg(sig, qp->rq.max_gs); 1482 } 1483 1484 qp->rq.wrid[ind] = wr->wr_id; 1485 1486 ind = (ind + 1) & (qp->rq.wqe_cnt - 1); 1487 } 1488 1489 out: 1490 if (likely(nreq)) { 1491 qp->rq.head += nreq; 1492 1493 /* Make sure that descriptors are written before 1494 * doorbell record. 1495 */ 1496 wmb(); 1497 1498 *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff); 1499 } 1500 1501 spin_unlock_irqrestore(&qp->rq.lock, flags); 1502 1503 return err; 1504 } 1505