1 /* 2 * Copyright (c) 2016-2017 Hisilicon Limited. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/acpi.h> 34 #include <linux/etherdevice.h> 35 #include <linux/interrupt.h> 36 #include <linux/kernel.h> 37 #include <linux/types.h> 38 #include <net/addrconf.h> 39 #include <rdma/ib_addr.h> 40 #include <rdma/ib_cache.h> 41 #include <rdma/ib_umem.h> 42 #include <rdma/uverbs_ioctl.h> 43 44 #include "hnae3.h" 45 #include "hns_roce_common.h" 46 #include "hns_roce_device.h" 47 #include "hns_roce_cmd.h" 48 #include "hns_roce_hem.h" 49 #include "hns_roce_hw_v2.h" 50 51 static void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg, 52 struct ib_sge *sg) 53 { 54 dseg->lkey = cpu_to_le32(sg->lkey); 55 dseg->addr = cpu_to_le64(sg->addr); 56 dseg->len = cpu_to_le32(sg->length); 57 } 58 59 static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, 60 struct hns_roce_wqe_frmr_seg *fseg, 61 const struct ib_reg_wr *wr) 62 { 63 struct hns_roce_mr *mr = to_hr_mr(wr->mr); 64 65 /* use ib_access_flags */ 66 roce_set_bit(rc_sq_wqe->byte_4, 67 V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S, 68 wr->access & IB_ACCESS_MW_BIND ? 1 : 0); 69 roce_set_bit(rc_sq_wqe->byte_4, 70 V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S, 71 wr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0); 72 roce_set_bit(rc_sq_wqe->byte_4, 73 V2_RC_FRMR_WQE_BYTE_4_RR_S, 74 wr->access & IB_ACCESS_REMOTE_READ ? 1 : 0); 75 roce_set_bit(rc_sq_wqe->byte_4, 76 V2_RC_FRMR_WQE_BYTE_4_RW_S, 77 wr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0); 78 roce_set_bit(rc_sq_wqe->byte_4, 79 V2_RC_FRMR_WQE_BYTE_4_LW_S, 80 wr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0); 81 82 /* Data structure reuse may lead to confusion */ 83 rc_sq_wqe->msg_len = cpu_to_le32(mr->pbl_ba & 0xffffffff); 84 rc_sq_wqe->inv_key = cpu_to_le32(mr->pbl_ba >> 32); 85 86 rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff); 87 rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32); 88 rc_sq_wqe->rkey = cpu_to_le32(wr->key); 89 rc_sq_wqe->va = cpu_to_le64(wr->mr->iova); 90 91 fseg->pbl_size = cpu_to_le32(mr->pbl_size); 92 roce_set_field(fseg->mode_buf_pg_sz, 93 V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M, 94 V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S, 95 mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET); 96 roce_set_bit(fseg->mode_buf_pg_sz, 97 V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0); 98 } 99 100 static void set_atomic_seg(struct hns_roce_wqe_atomic_seg *aseg, 101 const struct ib_atomic_wr *wr) 102 { 103 if (wr->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) { 104 aseg->fetchadd_swap_data = cpu_to_le64(wr->swap); 105 aseg->cmp_data = cpu_to_le64(wr->compare_add); 106 } else { 107 aseg->fetchadd_swap_data = cpu_to_le64(wr->compare_add); 108 aseg->cmp_data = 0; 109 } 110 } 111 112 static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr, 113 unsigned int *sge_ind) 114 { 115 struct hns_roce_v2_wqe_data_seg *dseg; 116 struct ib_sge *sg; 117 int num_in_wqe = 0; 118 int extend_sge_num; 119 int fi_sge_num; 120 int se_sge_num; 121 int shift; 122 int i; 123 124 if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) 125 num_in_wqe = HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE; 126 extend_sge_num = wr->num_sge - num_in_wqe; 127 sg = wr->sg_list + num_in_wqe; 128 shift = qp->hr_buf.page_shift; 129 130 /* 131 * Check whether wr->num_sge sges are in the same page. If not, we 132 * should calculate how many sges in the first page and the second 133 * page. 134 */ 135 dseg = get_send_extend_sge(qp, (*sge_ind) & (qp->sge.sge_cnt - 1)); 136 fi_sge_num = (round_up((uintptr_t)dseg, 1 << shift) - 137 (uintptr_t)dseg) / 138 sizeof(struct hns_roce_v2_wqe_data_seg); 139 if (extend_sge_num > fi_sge_num) { 140 se_sge_num = extend_sge_num - fi_sge_num; 141 for (i = 0; i < fi_sge_num; i++) { 142 set_data_seg_v2(dseg++, sg + i); 143 (*sge_ind)++; 144 } 145 dseg = get_send_extend_sge(qp, 146 (*sge_ind) & (qp->sge.sge_cnt - 1)); 147 for (i = 0; i < se_sge_num; i++) { 148 set_data_seg_v2(dseg++, sg + fi_sge_num + i); 149 (*sge_ind)++; 150 } 151 } else { 152 for (i = 0; i < extend_sge_num; i++) { 153 set_data_seg_v2(dseg++, sg + i); 154 (*sge_ind)++; 155 } 156 } 157 } 158 159 static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr, 160 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, 161 void *wqe, unsigned int *sge_ind, 162 const struct ib_send_wr **bad_wr) 163 { 164 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 165 struct hns_roce_v2_wqe_data_seg *dseg = wqe; 166 struct hns_roce_qp *qp = to_hr_qp(ibqp); 167 int i; 168 169 if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) { 170 if (le32_to_cpu(rc_sq_wqe->msg_len) > 171 hr_dev->caps.max_sq_inline) { 172 *bad_wr = wr; 173 dev_err(hr_dev->dev, "inline len(1-%d)=%d, illegal", 174 rc_sq_wqe->msg_len, hr_dev->caps.max_sq_inline); 175 return -EINVAL; 176 } 177 178 if (wr->opcode == IB_WR_RDMA_READ) { 179 *bad_wr = wr; 180 dev_err(hr_dev->dev, "Not support inline data!\n"); 181 return -EINVAL; 182 } 183 184 for (i = 0; i < wr->num_sge; i++) { 185 memcpy(wqe, ((void *)wr->sg_list[i].addr), 186 wr->sg_list[i].length); 187 wqe += wr->sg_list[i].length; 188 } 189 190 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S, 191 1); 192 } else { 193 if (wr->num_sge <= HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) { 194 for (i = 0; i < wr->num_sge; i++) { 195 if (likely(wr->sg_list[i].length)) { 196 set_data_seg_v2(dseg, wr->sg_list + i); 197 dseg++; 198 } 199 } 200 } else { 201 roce_set_field(rc_sq_wqe->byte_20, 202 V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M, 203 V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S, 204 (*sge_ind) & (qp->sge.sge_cnt - 1)); 205 206 for (i = 0; i < HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE; i++) { 207 if (likely(wr->sg_list[i].length)) { 208 set_data_seg_v2(dseg, wr->sg_list + i); 209 dseg++; 210 } 211 } 212 213 set_extend_sge(qp, wr, sge_ind); 214 } 215 216 roce_set_field(rc_sq_wqe->byte_16, 217 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M, 218 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, wr->num_sge); 219 } 220 221 return 0; 222 } 223 224 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, 225 const struct ib_qp_attr *attr, 226 int attr_mask, enum ib_qp_state cur_state, 227 enum ib_qp_state new_state); 228 229 static int hns_roce_v2_post_send(struct ib_qp *ibqp, 230 const struct ib_send_wr *wr, 231 const struct ib_send_wr **bad_wr) 232 { 233 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 234 struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah); 235 struct hns_roce_v2_ud_send_wqe *ud_sq_wqe; 236 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe; 237 struct hns_roce_qp *qp = to_hr_qp(ibqp); 238 struct hns_roce_wqe_frmr_seg *fseg; 239 struct device *dev = hr_dev->dev; 240 struct hns_roce_v2_db sq_db; 241 struct ib_qp_attr attr; 242 unsigned int sge_ind = 0; 243 unsigned int owner_bit; 244 unsigned long flags; 245 unsigned int ind; 246 void *wqe = NULL; 247 bool loopback; 248 int attr_mask; 249 u32 tmp_len; 250 int ret = 0; 251 u32 hr_op; 252 u8 *smac; 253 int nreq; 254 int i; 255 256 if (unlikely(ibqp->qp_type != IB_QPT_RC && 257 ibqp->qp_type != IB_QPT_GSI && 258 ibqp->qp_type != IB_QPT_UD)) { 259 dev_err(dev, "Not supported QP(0x%x)type!\n", ibqp->qp_type); 260 *bad_wr = wr; 261 return -EOPNOTSUPP; 262 } 263 264 if (unlikely(qp->state == IB_QPS_RESET || qp->state == IB_QPS_INIT || 265 qp->state == IB_QPS_RTR)) { 266 dev_err(dev, "Post WQE fail, QP state %d err!\n", qp->state); 267 *bad_wr = wr; 268 return -EINVAL; 269 } 270 271 spin_lock_irqsave(&qp->sq.lock, flags); 272 ind = qp->sq_next_wqe; 273 sge_ind = qp->next_sge; 274 275 for (nreq = 0; wr; ++nreq, wr = wr->next) { 276 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { 277 ret = -ENOMEM; 278 *bad_wr = wr; 279 goto out; 280 } 281 282 if (unlikely(wr->num_sge > qp->sq.max_gs)) { 283 dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n", 284 wr->num_sge, qp->sq.max_gs); 285 ret = -EINVAL; 286 *bad_wr = wr; 287 goto out; 288 } 289 290 wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1)); 291 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] = 292 wr->wr_id; 293 294 owner_bit = 295 ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1); 296 tmp_len = 0; 297 298 /* Corresponding to the QP type, wqe process separately */ 299 if (ibqp->qp_type == IB_QPT_GSI) { 300 ud_sq_wqe = wqe; 301 memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe)); 302 303 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M, 304 V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]); 305 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M, 306 V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]); 307 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M, 308 V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]); 309 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M, 310 V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]); 311 roce_set_field(ud_sq_wqe->byte_48, 312 V2_UD_SEND_WQE_BYTE_48_DMAC_4_M, 313 V2_UD_SEND_WQE_BYTE_48_DMAC_4_S, 314 ah->av.mac[4]); 315 roce_set_field(ud_sq_wqe->byte_48, 316 V2_UD_SEND_WQE_BYTE_48_DMAC_5_M, 317 V2_UD_SEND_WQE_BYTE_48_DMAC_5_S, 318 ah->av.mac[5]); 319 320 /* MAC loopback */ 321 smac = (u8 *)hr_dev->dev_addr[qp->port]; 322 loopback = ether_addr_equal_unaligned(ah->av.mac, 323 smac) ? 1 : 0; 324 325 roce_set_bit(ud_sq_wqe->byte_40, 326 V2_UD_SEND_WQE_BYTE_40_LBI_S, loopback); 327 328 roce_set_field(ud_sq_wqe->byte_4, 329 V2_UD_SEND_WQE_BYTE_4_OPCODE_M, 330 V2_UD_SEND_WQE_BYTE_4_OPCODE_S, 331 HNS_ROCE_V2_WQE_OP_SEND); 332 333 for (i = 0; i < wr->num_sge; i++) 334 tmp_len += wr->sg_list[i].length; 335 336 ud_sq_wqe->msg_len = 337 cpu_to_le32(le32_to_cpu(ud_sq_wqe->msg_len) + tmp_len); 338 339 switch (wr->opcode) { 340 case IB_WR_SEND_WITH_IMM: 341 case IB_WR_RDMA_WRITE_WITH_IMM: 342 ud_sq_wqe->immtdata = 343 cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); 344 break; 345 default: 346 ud_sq_wqe->immtdata = 0; 347 break; 348 } 349 350 /* Set sig attr */ 351 roce_set_bit(ud_sq_wqe->byte_4, 352 V2_UD_SEND_WQE_BYTE_4_CQE_S, 353 (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0); 354 355 /* Set se attr */ 356 roce_set_bit(ud_sq_wqe->byte_4, 357 V2_UD_SEND_WQE_BYTE_4_SE_S, 358 (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0); 359 360 roce_set_bit(ud_sq_wqe->byte_4, 361 V2_UD_SEND_WQE_BYTE_4_OWNER_S, owner_bit); 362 363 roce_set_field(ud_sq_wqe->byte_16, 364 V2_UD_SEND_WQE_BYTE_16_PD_M, 365 V2_UD_SEND_WQE_BYTE_16_PD_S, 366 to_hr_pd(ibqp->pd)->pdn); 367 368 roce_set_field(ud_sq_wqe->byte_16, 369 V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M, 370 V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S, 371 wr->num_sge); 372 373 roce_set_field(ud_sq_wqe->byte_20, 374 V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M, 375 V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S, 376 sge_ind & (qp->sge.sge_cnt - 1)); 377 378 roce_set_field(ud_sq_wqe->byte_24, 379 V2_UD_SEND_WQE_BYTE_24_UDPSPN_M, 380 V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, 0); 381 ud_sq_wqe->qkey = 382 cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ? 383 qp->qkey : ud_wr(wr)->remote_qkey); 384 roce_set_field(ud_sq_wqe->byte_32, 385 V2_UD_SEND_WQE_BYTE_32_DQPN_M, 386 V2_UD_SEND_WQE_BYTE_32_DQPN_S, 387 ud_wr(wr)->remote_qpn); 388 389 roce_set_field(ud_sq_wqe->byte_36, 390 V2_UD_SEND_WQE_BYTE_36_VLAN_M, 391 V2_UD_SEND_WQE_BYTE_36_VLAN_S, 392 le16_to_cpu(ah->av.vlan)); 393 roce_set_field(ud_sq_wqe->byte_36, 394 V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M, 395 V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S, 396 ah->av.hop_limit); 397 roce_set_field(ud_sq_wqe->byte_36, 398 V2_UD_SEND_WQE_BYTE_36_TCLASS_M, 399 V2_UD_SEND_WQE_BYTE_36_TCLASS_S, 400 ah->av.sl_tclass_flowlabel >> 401 HNS_ROCE_TCLASS_SHIFT); 402 roce_set_field(ud_sq_wqe->byte_40, 403 V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M, 404 V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S, 405 ah->av.sl_tclass_flowlabel & 406 HNS_ROCE_FLOW_LABEL_MASK); 407 roce_set_field(ud_sq_wqe->byte_40, 408 V2_UD_SEND_WQE_BYTE_40_SL_M, 409 V2_UD_SEND_WQE_BYTE_40_SL_S, 410 le32_to_cpu(ah->av.sl_tclass_flowlabel) >> 411 HNS_ROCE_SL_SHIFT); 412 roce_set_field(ud_sq_wqe->byte_40, 413 V2_UD_SEND_WQE_BYTE_40_PORTN_M, 414 V2_UD_SEND_WQE_BYTE_40_PORTN_S, 415 qp->port); 416 417 roce_set_bit(ud_sq_wqe->byte_40, 418 V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S, 419 ah->av.vlan_en ? 1 : 0); 420 roce_set_field(ud_sq_wqe->byte_48, 421 V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M, 422 V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S, 423 hns_get_gid_index(hr_dev, qp->phy_port, 424 ah->av.gid_index)); 425 426 memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], 427 GID_LEN_V2); 428 429 set_extend_sge(qp, wr, &sge_ind); 430 ind++; 431 } else if (ibqp->qp_type == IB_QPT_RC) { 432 rc_sq_wqe = wqe; 433 memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe)); 434 for (i = 0; i < wr->num_sge; i++) 435 tmp_len += wr->sg_list[i].length; 436 437 rc_sq_wqe->msg_len = 438 cpu_to_le32(le32_to_cpu(rc_sq_wqe->msg_len) + tmp_len); 439 440 switch (wr->opcode) { 441 case IB_WR_SEND_WITH_IMM: 442 case IB_WR_RDMA_WRITE_WITH_IMM: 443 rc_sq_wqe->immtdata = 444 cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); 445 break; 446 case IB_WR_SEND_WITH_INV: 447 rc_sq_wqe->inv_key = 448 cpu_to_le32(wr->ex.invalidate_rkey); 449 break; 450 default: 451 rc_sq_wqe->immtdata = 0; 452 break; 453 } 454 455 roce_set_bit(rc_sq_wqe->byte_4, 456 V2_RC_SEND_WQE_BYTE_4_FENCE_S, 457 (wr->send_flags & IB_SEND_FENCE) ? 1 : 0); 458 459 roce_set_bit(rc_sq_wqe->byte_4, 460 V2_RC_SEND_WQE_BYTE_4_SE_S, 461 (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0); 462 463 roce_set_bit(rc_sq_wqe->byte_4, 464 V2_RC_SEND_WQE_BYTE_4_CQE_S, 465 (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0); 466 467 roce_set_bit(rc_sq_wqe->byte_4, 468 V2_RC_SEND_WQE_BYTE_4_OWNER_S, owner_bit); 469 470 wqe += sizeof(struct hns_roce_v2_rc_send_wqe); 471 switch (wr->opcode) { 472 case IB_WR_RDMA_READ: 473 hr_op = HNS_ROCE_V2_WQE_OP_RDMA_READ; 474 rc_sq_wqe->rkey = 475 cpu_to_le32(rdma_wr(wr)->rkey); 476 rc_sq_wqe->va = 477 cpu_to_le64(rdma_wr(wr)->remote_addr); 478 break; 479 case IB_WR_RDMA_WRITE: 480 hr_op = HNS_ROCE_V2_WQE_OP_RDMA_WRITE; 481 rc_sq_wqe->rkey = 482 cpu_to_le32(rdma_wr(wr)->rkey); 483 rc_sq_wqe->va = 484 cpu_to_le64(rdma_wr(wr)->remote_addr); 485 break; 486 case IB_WR_RDMA_WRITE_WITH_IMM: 487 hr_op = HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM; 488 rc_sq_wqe->rkey = 489 cpu_to_le32(rdma_wr(wr)->rkey); 490 rc_sq_wqe->va = 491 cpu_to_le64(rdma_wr(wr)->remote_addr); 492 break; 493 case IB_WR_SEND: 494 hr_op = HNS_ROCE_V2_WQE_OP_SEND; 495 break; 496 case IB_WR_SEND_WITH_INV: 497 hr_op = HNS_ROCE_V2_WQE_OP_SEND_WITH_INV; 498 break; 499 case IB_WR_SEND_WITH_IMM: 500 hr_op = HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM; 501 break; 502 case IB_WR_LOCAL_INV: 503 hr_op = HNS_ROCE_V2_WQE_OP_LOCAL_INV; 504 roce_set_bit(rc_sq_wqe->byte_4, 505 V2_RC_SEND_WQE_BYTE_4_SO_S, 1); 506 rc_sq_wqe->inv_key = 507 cpu_to_le32(wr->ex.invalidate_rkey); 508 break; 509 case IB_WR_REG_MR: 510 hr_op = HNS_ROCE_V2_WQE_OP_FAST_REG_PMR; 511 fseg = wqe; 512 set_frmr_seg(rc_sq_wqe, fseg, reg_wr(wr)); 513 break; 514 case IB_WR_ATOMIC_CMP_AND_SWP: 515 hr_op = HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP; 516 rc_sq_wqe->rkey = 517 cpu_to_le32(atomic_wr(wr)->rkey); 518 rc_sq_wqe->va = 519 cpu_to_le64(atomic_wr(wr)->remote_addr); 520 break; 521 case IB_WR_ATOMIC_FETCH_AND_ADD: 522 hr_op = HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD; 523 rc_sq_wqe->rkey = 524 cpu_to_le32(atomic_wr(wr)->rkey); 525 rc_sq_wqe->va = 526 cpu_to_le64(atomic_wr(wr)->remote_addr); 527 break; 528 case IB_WR_MASKED_ATOMIC_CMP_AND_SWP: 529 hr_op = 530 HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP; 531 break; 532 case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD: 533 hr_op = 534 HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD; 535 break; 536 default: 537 hr_op = HNS_ROCE_V2_WQE_OP_MASK; 538 break; 539 } 540 541 roce_set_field(rc_sq_wqe->byte_4, 542 V2_RC_SEND_WQE_BYTE_4_OPCODE_M, 543 V2_RC_SEND_WQE_BYTE_4_OPCODE_S, hr_op); 544 545 if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || 546 wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 547 struct hns_roce_v2_wqe_data_seg *dseg; 548 549 dseg = wqe; 550 set_data_seg_v2(dseg, wr->sg_list); 551 wqe += sizeof(struct hns_roce_v2_wqe_data_seg); 552 set_atomic_seg(wqe, atomic_wr(wr)); 553 roce_set_field(rc_sq_wqe->byte_16, 554 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M, 555 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, 556 wr->num_sge); 557 } else if (wr->opcode != IB_WR_REG_MR) { 558 ret = set_rwqe_data_seg(ibqp, wr, rc_sq_wqe, 559 wqe, &sge_ind, bad_wr); 560 if (ret) 561 goto out; 562 } 563 564 ind++; 565 } else { 566 dev_err(dev, "Illegal qp_type(0x%x)\n", ibqp->qp_type); 567 spin_unlock_irqrestore(&qp->sq.lock, flags); 568 *bad_wr = wr; 569 return -EOPNOTSUPP; 570 } 571 } 572 573 out: 574 if (likely(nreq)) { 575 qp->sq.head += nreq; 576 /* Memory barrier */ 577 wmb(); 578 579 sq_db.byte_4 = 0; 580 sq_db.parameter = 0; 581 582 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M, 583 V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn); 584 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M, 585 V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB); 586 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M, 587 V2_DB_PARAMETER_IDX_S, 588 qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1)); 589 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M, 590 V2_DB_PARAMETER_SL_S, qp->sl); 591 592 hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg_l); 593 594 qp->sq_next_wqe = ind; 595 qp->next_sge = sge_ind; 596 597 if (qp->state == IB_QPS_ERR) { 598 attr_mask = IB_QP_STATE; 599 attr.qp_state = IB_QPS_ERR; 600 601 ret = hns_roce_v2_modify_qp(&qp->ibqp, &attr, attr_mask, 602 qp->state, IB_QPS_ERR); 603 if (ret) { 604 spin_unlock_irqrestore(&qp->sq.lock, flags); 605 *bad_wr = wr; 606 return ret; 607 } 608 } 609 } 610 611 spin_unlock_irqrestore(&qp->sq.lock, flags); 612 613 return ret; 614 } 615 616 static int hns_roce_v2_post_recv(struct ib_qp *ibqp, 617 const struct ib_recv_wr *wr, 618 const struct ib_recv_wr **bad_wr) 619 { 620 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 621 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 622 struct hns_roce_v2_wqe_data_seg *dseg; 623 struct hns_roce_rinl_sge *sge_list; 624 struct device *dev = hr_dev->dev; 625 struct ib_qp_attr attr; 626 unsigned long flags; 627 void *wqe = NULL; 628 int attr_mask; 629 int ret = 0; 630 int nreq; 631 int ind; 632 int i; 633 634 spin_lock_irqsave(&hr_qp->rq.lock, flags); 635 ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1); 636 637 if (hr_qp->state == IB_QPS_RESET) { 638 spin_unlock_irqrestore(&hr_qp->rq.lock, flags); 639 *bad_wr = wr; 640 return -EINVAL; 641 } 642 643 for (nreq = 0; wr; ++nreq, wr = wr->next) { 644 if (hns_roce_wq_overflow(&hr_qp->rq, nreq, 645 hr_qp->ibqp.recv_cq)) { 646 ret = -ENOMEM; 647 *bad_wr = wr; 648 goto out; 649 } 650 651 if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) { 652 dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n", 653 wr->num_sge, hr_qp->rq.max_gs); 654 ret = -EINVAL; 655 *bad_wr = wr; 656 goto out; 657 } 658 659 wqe = get_recv_wqe(hr_qp, ind); 660 dseg = (struct hns_roce_v2_wqe_data_seg *)wqe; 661 for (i = 0; i < wr->num_sge; i++) { 662 if (!wr->sg_list[i].length) 663 continue; 664 set_data_seg_v2(dseg, wr->sg_list + i); 665 dseg++; 666 } 667 668 if (i < hr_qp->rq.max_gs) { 669 dseg->lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY); 670 dseg->addr = 0; 671 } 672 673 /* rq support inline data */ 674 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) { 675 sge_list = hr_qp->rq_inl_buf.wqe_list[ind].sg_list; 676 hr_qp->rq_inl_buf.wqe_list[ind].sge_cnt = 677 (u32)wr->num_sge; 678 for (i = 0; i < wr->num_sge; i++) { 679 sge_list[i].addr = 680 (void *)(u64)wr->sg_list[i].addr; 681 sge_list[i].len = wr->sg_list[i].length; 682 } 683 } 684 685 hr_qp->rq.wrid[ind] = wr->wr_id; 686 687 ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1); 688 } 689 690 out: 691 if (likely(nreq)) { 692 hr_qp->rq.head += nreq; 693 /* Memory barrier */ 694 wmb(); 695 696 *hr_qp->rdb.db_record = hr_qp->rq.head & 0xffff; 697 698 if (hr_qp->state == IB_QPS_ERR) { 699 attr_mask = IB_QP_STATE; 700 attr.qp_state = IB_QPS_ERR; 701 702 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, &attr, 703 attr_mask, hr_qp->state, 704 IB_QPS_ERR); 705 if (ret) { 706 spin_unlock_irqrestore(&hr_qp->rq.lock, flags); 707 *bad_wr = wr; 708 return ret; 709 } 710 } 711 } 712 spin_unlock_irqrestore(&hr_qp->rq.lock, flags); 713 714 return ret; 715 } 716 717 static int hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev, 718 unsigned long instance_stage, 719 unsigned long reset_stage) 720 { 721 /* When hardware reset has been completed once or more, we should stop 722 * sending mailbox&cmq&doorbell to hardware. If now in .init_instance() 723 * function, we should exit with error. If now at HNAE3_INIT_CLIENT 724 * stage of soft reset process, we should exit with error, and then 725 * HNAE3_INIT_CLIENT related process can rollback the operation like 726 * notifing hardware to free resources, HNAE3_INIT_CLIENT related 727 * process will exit with error to notify NIC driver to reschedule soft 728 * reset process once again. 729 */ 730 hr_dev->is_reset = true; 731 hr_dev->dis_db = true; 732 733 if (reset_stage == HNS_ROCE_STATE_RST_INIT || 734 instance_stage == HNS_ROCE_STATE_INIT) 735 return CMD_RST_PRC_EBUSY; 736 737 return CMD_RST_PRC_SUCCESS; 738 } 739 740 static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev, 741 unsigned long instance_stage, 742 unsigned long reset_stage) 743 { 744 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 745 struct hnae3_handle *handle = priv->handle; 746 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 747 748 /* When hardware reset is detected, we should stop sending mailbox&cmq& 749 * doorbell to hardware. If now in .init_instance() function, we should 750 * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset 751 * process, we should exit with error, and then HNAE3_INIT_CLIENT 752 * related process can rollback the operation like notifing hardware to 753 * free resources, HNAE3_INIT_CLIENT related process will exit with 754 * error to notify NIC driver to reschedule soft reset process once 755 * again. 756 */ 757 hr_dev->dis_db = true; 758 if (!ops->get_hw_reset_stat(handle)) 759 hr_dev->is_reset = true; 760 761 if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT || 762 instance_stage == HNS_ROCE_STATE_INIT) 763 return CMD_RST_PRC_EBUSY; 764 765 return CMD_RST_PRC_SUCCESS; 766 } 767 768 static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev) 769 { 770 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 771 struct hnae3_handle *handle = priv->handle; 772 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 773 774 /* When software reset is detected at .init_instance() function, we 775 * should stop sending mailbox&cmq&doorbell to hardware, and exit 776 * with error. 777 */ 778 hr_dev->dis_db = true; 779 if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt) 780 hr_dev->is_reset = true; 781 782 return CMD_RST_PRC_EBUSY; 783 } 784 785 static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev) 786 { 787 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 788 struct hnae3_handle *handle = priv->handle; 789 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 790 unsigned long instance_stage; /* the current instance stage */ 791 unsigned long reset_stage; /* the current reset stage */ 792 unsigned long reset_cnt; 793 bool sw_resetting; 794 bool hw_resetting; 795 796 if (hr_dev->is_reset) 797 return CMD_RST_PRC_SUCCESS; 798 799 /* Get information about reset from NIC driver or RoCE driver itself, 800 * the meaning of the following variables from NIC driver are described 801 * as below: 802 * reset_cnt -- The count value of completed hardware reset. 803 * hw_resetting -- Whether hardware device is resetting now. 804 * sw_resetting -- Whether NIC's software reset process is running now. 805 */ 806 instance_stage = handle->rinfo.instance_state; 807 reset_stage = handle->rinfo.reset_state; 808 reset_cnt = ops->ae_dev_reset_cnt(handle); 809 hw_resetting = ops->get_hw_reset_stat(handle); 810 sw_resetting = ops->ae_dev_resetting(handle); 811 812 if (reset_cnt != hr_dev->reset_cnt) 813 return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage, 814 reset_stage); 815 else if (hw_resetting) 816 return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage, 817 reset_stage); 818 else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT) 819 return hns_roce_v2_cmd_sw_resetting(hr_dev); 820 821 return 0; 822 } 823 824 static int hns_roce_cmq_space(struct hns_roce_v2_cmq_ring *ring) 825 { 826 int ntu = ring->next_to_use; 827 int ntc = ring->next_to_clean; 828 int used = (ntu - ntc + ring->desc_num) % ring->desc_num; 829 830 return ring->desc_num - used - 1; 831 } 832 833 static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev, 834 struct hns_roce_v2_cmq_ring *ring) 835 { 836 int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc); 837 838 ring->desc = kzalloc(size, GFP_KERNEL); 839 if (!ring->desc) 840 return -ENOMEM; 841 842 ring->desc_dma_addr = dma_map_single(hr_dev->dev, ring->desc, size, 843 DMA_BIDIRECTIONAL); 844 if (dma_mapping_error(hr_dev->dev, ring->desc_dma_addr)) { 845 ring->desc_dma_addr = 0; 846 kfree(ring->desc); 847 ring->desc = NULL; 848 return -ENOMEM; 849 } 850 851 return 0; 852 } 853 854 static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev, 855 struct hns_roce_v2_cmq_ring *ring) 856 { 857 dma_unmap_single(hr_dev->dev, ring->desc_dma_addr, 858 ring->desc_num * sizeof(struct hns_roce_cmq_desc), 859 DMA_BIDIRECTIONAL); 860 861 ring->desc_dma_addr = 0; 862 kfree(ring->desc); 863 } 864 865 static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type) 866 { 867 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 868 struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ? 869 &priv->cmq.csq : &priv->cmq.crq; 870 871 ring->flag = ring_type; 872 ring->next_to_clean = 0; 873 ring->next_to_use = 0; 874 875 return hns_roce_alloc_cmq_desc(hr_dev, ring); 876 } 877 878 static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type) 879 { 880 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 881 struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ? 882 &priv->cmq.csq : &priv->cmq.crq; 883 dma_addr_t dma = ring->desc_dma_addr; 884 885 if (ring_type == TYPE_CSQ) { 886 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, (u32)dma); 887 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG, 888 upper_32_bits(dma)); 889 roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG, 890 (ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S) | 891 HNS_ROCE_CMQ_ENABLE); 892 roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0); 893 roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0); 894 } else { 895 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_L_REG, (u32)dma); 896 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG, 897 upper_32_bits(dma)); 898 roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG, 899 (ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S) | 900 HNS_ROCE_CMQ_ENABLE); 901 roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0); 902 roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0); 903 } 904 } 905 906 static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev) 907 { 908 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 909 int ret; 910 911 /* Setup the queue entries for command queue */ 912 priv->cmq.csq.desc_num = CMD_CSQ_DESC_NUM; 913 priv->cmq.crq.desc_num = CMD_CRQ_DESC_NUM; 914 915 /* Setup the lock for command queue */ 916 spin_lock_init(&priv->cmq.csq.lock); 917 spin_lock_init(&priv->cmq.crq.lock); 918 919 /* Setup Tx write back timeout */ 920 priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT; 921 922 /* Init CSQ */ 923 ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CSQ); 924 if (ret) { 925 dev_err(hr_dev->dev, "Init CSQ error, ret = %d.\n", ret); 926 return ret; 927 } 928 929 /* Init CRQ */ 930 ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CRQ); 931 if (ret) { 932 dev_err(hr_dev->dev, "Init CRQ error, ret = %d.\n", ret); 933 goto err_crq; 934 } 935 936 /* Init CSQ REG */ 937 hns_roce_cmq_init_regs(hr_dev, TYPE_CSQ); 938 939 /* Init CRQ REG */ 940 hns_roce_cmq_init_regs(hr_dev, TYPE_CRQ); 941 942 return 0; 943 944 err_crq: 945 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq); 946 947 return ret; 948 } 949 950 static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev) 951 { 952 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 953 954 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq); 955 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq); 956 } 957 958 static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc, 959 enum hns_roce_opcode_type opcode, 960 bool is_read) 961 { 962 memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc)); 963 desc->opcode = cpu_to_le16(opcode); 964 desc->flag = 965 cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN); 966 if (is_read) 967 desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR); 968 else 969 desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR); 970 } 971 972 static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev) 973 { 974 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 975 u32 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG); 976 977 return head == priv->cmq.csq.next_to_use; 978 } 979 980 static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev) 981 { 982 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 983 struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq; 984 struct hns_roce_cmq_desc *desc; 985 u16 ntc = csq->next_to_clean; 986 u32 head; 987 int clean = 0; 988 989 desc = &csq->desc[ntc]; 990 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG); 991 while (head != ntc) { 992 memset(desc, 0, sizeof(*desc)); 993 ntc++; 994 if (ntc == csq->desc_num) 995 ntc = 0; 996 desc = &csq->desc[ntc]; 997 clean++; 998 } 999 csq->next_to_clean = ntc; 1000 1001 return clean; 1002 } 1003 1004 static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev, 1005 struct hns_roce_cmq_desc *desc, int num) 1006 { 1007 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 1008 struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq; 1009 struct hns_roce_cmq_desc *desc_to_use; 1010 bool complete = false; 1011 u32 timeout = 0; 1012 int handle = 0; 1013 u16 desc_ret; 1014 int ret = 0; 1015 int ntc; 1016 1017 spin_lock_bh(&csq->lock); 1018 1019 if (num > hns_roce_cmq_space(csq)) { 1020 spin_unlock_bh(&csq->lock); 1021 return -EBUSY; 1022 } 1023 1024 /* 1025 * Record the location of desc in the cmq for this time 1026 * which will be use for hardware to write back 1027 */ 1028 ntc = csq->next_to_use; 1029 1030 while (handle < num) { 1031 desc_to_use = &csq->desc[csq->next_to_use]; 1032 *desc_to_use = desc[handle]; 1033 dev_dbg(hr_dev->dev, "set cmq desc:\n"); 1034 csq->next_to_use++; 1035 if (csq->next_to_use == csq->desc_num) 1036 csq->next_to_use = 0; 1037 handle++; 1038 } 1039 1040 /* Write to hardware */ 1041 roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, csq->next_to_use); 1042 1043 /* 1044 * If the command is sync, wait for the firmware to write back, 1045 * if multi descriptors to be sent, use the first one to check 1046 */ 1047 if ((desc->flag) & HNS_ROCE_CMD_FLAG_NO_INTR) { 1048 do { 1049 if (hns_roce_cmq_csq_done(hr_dev)) 1050 break; 1051 udelay(1); 1052 timeout++; 1053 } while (timeout < priv->cmq.tx_timeout); 1054 } 1055 1056 if (hns_roce_cmq_csq_done(hr_dev)) { 1057 complete = true; 1058 handle = 0; 1059 while (handle < num) { 1060 /* get the result of hardware write back */ 1061 desc_to_use = &csq->desc[ntc]; 1062 desc[handle] = *desc_to_use; 1063 dev_dbg(hr_dev->dev, "Get cmq desc:\n"); 1064 desc_ret = desc[handle].retval; 1065 if (desc_ret == CMD_EXEC_SUCCESS) 1066 ret = 0; 1067 else 1068 ret = -EIO; 1069 priv->cmq.last_status = desc_ret; 1070 ntc++; 1071 handle++; 1072 if (ntc == csq->desc_num) 1073 ntc = 0; 1074 } 1075 } 1076 1077 if (!complete) 1078 ret = -EAGAIN; 1079 1080 /* clean the command send queue */ 1081 handle = hns_roce_cmq_csq_clean(hr_dev); 1082 if (handle != num) 1083 dev_warn(hr_dev->dev, "Cleaned %d, need to clean %d\n", 1084 handle, num); 1085 1086 spin_unlock_bh(&csq->lock); 1087 1088 return ret; 1089 } 1090 1091 static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev, 1092 struct hns_roce_cmq_desc *desc, int num) 1093 { 1094 int retval; 1095 int ret; 1096 1097 ret = hns_roce_v2_rst_process_cmd(hr_dev); 1098 if (ret == CMD_RST_PRC_SUCCESS) 1099 return 0; 1100 if (ret == CMD_RST_PRC_EBUSY) 1101 return -EBUSY; 1102 1103 ret = __hns_roce_cmq_send(hr_dev, desc, num); 1104 if (ret) { 1105 retval = hns_roce_v2_rst_process_cmd(hr_dev); 1106 if (retval == CMD_RST_PRC_SUCCESS) 1107 return 0; 1108 else if (retval == CMD_RST_PRC_EBUSY) 1109 return -EBUSY; 1110 } 1111 1112 return ret; 1113 } 1114 1115 static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev) 1116 { 1117 struct hns_roce_query_version *resp; 1118 struct hns_roce_cmq_desc desc; 1119 int ret; 1120 1121 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true); 1122 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1123 if (ret) 1124 return ret; 1125 1126 resp = (struct hns_roce_query_version *)desc.data; 1127 hr_dev->hw_rev = le32_to_cpu(resp->rocee_hw_version); 1128 hr_dev->vendor_id = hr_dev->pci_dev->vendor; 1129 1130 return 0; 1131 } 1132 1133 static void hns_roce_function_clear(struct hns_roce_dev *hr_dev) 1134 { 1135 struct hns_roce_func_clear *resp; 1136 struct hns_roce_cmq_desc desc; 1137 unsigned long end; 1138 int ret; 1139 1140 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false); 1141 resp = (struct hns_roce_func_clear *)desc.data; 1142 1143 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1144 if (ret) { 1145 dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n", 1146 ret); 1147 return; 1148 } 1149 1150 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL); 1151 end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS; 1152 while (end) { 1153 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT); 1154 end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT; 1155 1156 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, 1157 true); 1158 1159 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1160 if (ret) 1161 continue; 1162 1163 if (roce_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) { 1164 hr_dev->is_reset = true; 1165 return; 1166 } 1167 } 1168 1169 dev_err(hr_dev->dev, "Func clear fail.\n"); 1170 } 1171 1172 static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev) 1173 { 1174 struct hns_roce_query_fw_info *resp; 1175 struct hns_roce_cmq_desc desc; 1176 int ret; 1177 1178 hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true); 1179 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1180 if (ret) 1181 return ret; 1182 1183 resp = (struct hns_roce_query_fw_info *)desc.data; 1184 hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver)); 1185 1186 return 0; 1187 } 1188 1189 static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev) 1190 { 1191 struct hns_roce_cfg_global_param *req; 1192 struct hns_roce_cmq_desc desc; 1193 1194 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM, 1195 false); 1196 1197 req = (struct hns_roce_cfg_global_param *)desc.data; 1198 memset(req, 0, sizeof(*req)); 1199 roce_set_field(req->time_cfg_udp_port, 1200 CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M, 1201 CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S, 0x3e8); 1202 roce_set_field(req->time_cfg_udp_port, 1203 CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M, 1204 CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S, 0x12b7); 1205 1206 return hns_roce_cmq_send(hr_dev, &desc, 1); 1207 } 1208 1209 static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev) 1210 { 1211 struct hns_roce_cmq_desc desc[2]; 1212 struct hns_roce_pf_res_a *req_a; 1213 struct hns_roce_pf_res_b *req_b; 1214 int ret; 1215 int i; 1216 1217 for (i = 0; i < 2; i++) { 1218 hns_roce_cmq_setup_basic_desc(&desc[i], 1219 HNS_ROCE_OPC_QUERY_PF_RES, true); 1220 1221 if (i == 0) 1222 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1223 else 1224 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1225 } 1226 1227 ret = hns_roce_cmq_send(hr_dev, desc, 2); 1228 if (ret) 1229 return ret; 1230 1231 req_a = (struct hns_roce_pf_res_a *)desc[0].data; 1232 req_b = (struct hns_roce_pf_res_b *)desc[1].data; 1233 1234 hr_dev->caps.qpc_bt_num = roce_get_field(req_a->qpc_bt_idx_num, 1235 PF_RES_DATA_1_PF_QPC_BT_NUM_M, 1236 PF_RES_DATA_1_PF_QPC_BT_NUM_S); 1237 hr_dev->caps.srqc_bt_num = roce_get_field(req_a->srqc_bt_idx_num, 1238 PF_RES_DATA_2_PF_SRQC_BT_NUM_M, 1239 PF_RES_DATA_2_PF_SRQC_BT_NUM_S); 1240 hr_dev->caps.cqc_bt_num = roce_get_field(req_a->cqc_bt_idx_num, 1241 PF_RES_DATA_3_PF_CQC_BT_NUM_M, 1242 PF_RES_DATA_3_PF_CQC_BT_NUM_S); 1243 hr_dev->caps.mpt_bt_num = roce_get_field(req_a->mpt_bt_idx_num, 1244 PF_RES_DATA_4_PF_MPT_BT_NUM_M, 1245 PF_RES_DATA_4_PF_MPT_BT_NUM_S); 1246 1247 hr_dev->caps.sl_num = roce_get_field(req_b->qid_idx_sl_num, 1248 PF_RES_DATA_3_PF_SL_NUM_M, 1249 PF_RES_DATA_3_PF_SL_NUM_S); 1250 hr_dev->caps.sccc_bt_num = roce_get_field(req_b->sccc_bt_idx_num, 1251 PF_RES_DATA_4_PF_SCCC_BT_NUM_M, 1252 PF_RES_DATA_4_PF_SCCC_BT_NUM_S); 1253 1254 return 0; 1255 } 1256 1257 static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev) 1258 { 1259 struct hns_roce_pf_timer_res_a *req_a; 1260 struct hns_roce_cmq_desc desc[2]; 1261 int ret, i; 1262 1263 for (i = 0; i < 2; i++) { 1264 hns_roce_cmq_setup_basic_desc(&desc[i], 1265 HNS_ROCE_OPC_QUERY_PF_TIMER_RES, 1266 true); 1267 1268 if (i == 0) 1269 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1270 else 1271 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1272 } 1273 1274 ret = hns_roce_cmq_send(hr_dev, desc, 2); 1275 if (ret) 1276 return ret; 1277 1278 req_a = (struct hns_roce_pf_timer_res_a *)desc[0].data; 1279 1280 hr_dev->caps.qpc_timer_bt_num = 1281 roce_get_field(req_a->qpc_timer_bt_idx_num, 1282 PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M, 1283 PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S); 1284 hr_dev->caps.cqc_timer_bt_num = 1285 roce_get_field(req_a->cqc_timer_bt_idx_num, 1286 PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M, 1287 PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S); 1288 1289 return 0; 1290 } 1291 1292 static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, 1293 int vf_id) 1294 { 1295 struct hns_roce_cmq_desc desc; 1296 struct hns_roce_vf_switch *swt; 1297 int ret; 1298 1299 swt = (struct hns_roce_vf_switch *)desc.data; 1300 hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true); 1301 swt->rocee_sel |= cpu_to_le16(HNS_ICL_SWITCH_CMD_ROCEE_SEL); 1302 roce_set_field(swt->fun_id, 1303 VF_SWITCH_DATA_FUN_ID_VF_ID_M, 1304 VF_SWITCH_DATA_FUN_ID_VF_ID_S, 1305 vf_id); 1306 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1307 if (ret) 1308 return ret; 1309 desc.flag = 1310 cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN); 1311 desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR); 1312 roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LPBK_S, 1); 1313 roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S, 1); 1314 roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S, 1); 1315 1316 return hns_roce_cmq_send(hr_dev, &desc, 1); 1317 } 1318 1319 static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev) 1320 { 1321 struct hns_roce_cmq_desc desc[2]; 1322 struct hns_roce_vf_res_a *req_a; 1323 struct hns_roce_vf_res_b *req_b; 1324 int i; 1325 1326 req_a = (struct hns_roce_vf_res_a *)desc[0].data; 1327 req_b = (struct hns_roce_vf_res_b *)desc[1].data; 1328 memset(req_a, 0, sizeof(*req_a)); 1329 memset(req_b, 0, sizeof(*req_b)); 1330 for (i = 0; i < 2; i++) { 1331 hns_roce_cmq_setup_basic_desc(&desc[i], 1332 HNS_ROCE_OPC_ALLOC_VF_RES, false); 1333 1334 if (i == 0) 1335 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1336 else 1337 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1338 1339 if (i == 0) { 1340 roce_set_field(req_a->vf_qpc_bt_idx_num, 1341 VF_RES_A_DATA_1_VF_QPC_BT_IDX_M, 1342 VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0); 1343 roce_set_field(req_a->vf_qpc_bt_idx_num, 1344 VF_RES_A_DATA_1_VF_QPC_BT_NUM_M, 1345 VF_RES_A_DATA_1_VF_QPC_BT_NUM_S, 1346 HNS_ROCE_VF_QPC_BT_NUM); 1347 1348 roce_set_field(req_a->vf_srqc_bt_idx_num, 1349 VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M, 1350 VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0); 1351 roce_set_field(req_a->vf_srqc_bt_idx_num, 1352 VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M, 1353 VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S, 1354 HNS_ROCE_VF_SRQC_BT_NUM); 1355 1356 roce_set_field(req_a->vf_cqc_bt_idx_num, 1357 VF_RES_A_DATA_3_VF_CQC_BT_IDX_M, 1358 VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0); 1359 roce_set_field(req_a->vf_cqc_bt_idx_num, 1360 VF_RES_A_DATA_3_VF_CQC_BT_NUM_M, 1361 VF_RES_A_DATA_3_VF_CQC_BT_NUM_S, 1362 HNS_ROCE_VF_CQC_BT_NUM); 1363 1364 roce_set_field(req_a->vf_mpt_bt_idx_num, 1365 VF_RES_A_DATA_4_VF_MPT_BT_IDX_M, 1366 VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0); 1367 roce_set_field(req_a->vf_mpt_bt_idx_num, 1368 VF_RES_A_DATA_4_VF_MPT_BT_NUM_M, 1369 VF_RES_A_DATA_4_VF_MPT_BT_NUM_S, 1370 HNS_ROCE_VF_MPT_BT_NUM); 1371 1372 roce_set_field(req_a->vf_eqc_bt_idx_num, 1373 VF_RES_A_DATA_5_VF_EQC_IDX_M, 1374 VF_RES_A_DATA_5_VF_EQC_IDX_S, 0); 1375 roce_set_field(req_a->vf_eqc_bt_idx_num, 1376 VF_RES_A_DATA_5_VF_EQC_NUM_M, 1377 VF_RES_A_DATA_5_VF_EQC_NUM_S, 1378 HNS_ROCE_VF_EQC_NUM); 1379 } else { 1380 roce_set_field(req_b->vf_smac_idx_num, 1381 VF_RES_B_DATA_1_VF_SMAC_IDX_M, 1382 VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0); 1383 roce_set_field(req_b->vf_smac_idx_num, 1384 VF_RES_B_DATA_1_VF_SMAC_NUM_M, 1385 VF_RES_B_DATA_1_VF_SMAC_NUM_S, 1386 HNS_ROCE_VF_SMAC_NUM); 1387 1388 roce_set_field(req_b->vf_sgid_idx_num, 1389 VF_RES_B_DATA_2_VF_SGID_IDX_M, 1390 VF_RES_B_DATA_2_VF_SGID_IDX_S, 0); 1391 roce_set_field(req_b->vf_sgid_idx_num, 1392 VF_RES_B_DATA_2_VF_SGID_NUM_M, 1393 VF_RES_B_DATA_2_VF_SGID_NUM_S, 1394 HNS_ROCE_VF_SGID_NUM); 1395 1396 roce_set_field(req_b->vf_qid_idx_sl_num, 1397 VF_RES_B_DATA_3_VF_QID_IDX_M, 1398 VF_RES_B_DATA_3_VF_QID_IDX_S, 0); 1399 roce_set_field(req_b->vf_qid_idx_sl_num, 1400 VF_RES_B_DATA_3_VF_SL_NUM_M, 1401 VF_RES_B_DATA_3_VF_SL_NUM_S, 1402 HNS_ROCE_VF_SL_NUM); 1403 1404 roce_set_field(req_b->vf_sccc_idx_num, 1405 VF_RES_B_DATA_4_VF_SCCC_BT_IDX_M, 1406 VF_RES_B_DATA_4_VF_SCCC_BT_IDX_S, 0); 1407 roce_set_field(req_b->vf_sccc_idx_num, 1408 VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M, 1409 VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S, 1410 HNS_ROCE_VF_SCCC_BT_NUM); 1411 } 1412 } 1413 1414 return hns_roce_cmq_send(hr_dev, desc, 2); 1415 } 1416 1417 static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev) 1418 { 1419 u8 srqc_hop_num = hr_dev->caps.srqc_hop_num; 1420 u8 qpc_hop_num = hr_dev->caps.qpc_hop_num; 1421 u8 cqc_hop_num = hr_dev->caps.cqc_hop_num; 1422 u8 mpt_hop_num = hr_dev->caps.mpt_hop_num; 1423 u8 sccc_hop_num = hr_dev->caps.sccc_hop_num; 1424 struct hns_roce_cfg_bt_attr *req; 1425 struct hns_roce_cmq_desc desc; 1426 1427 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false); 1428 req = (struct hns_roce_cfg_bt_attr *)desc.data; 1429 memset(req, 0, sizeof(*req)); 1430 1431 roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M, 1432 CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S, 1433 hr_dev->caps.qpc_ba_pg_sz + PG_SHIFT_OFFSET); 1434 roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M, 1435 CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S, 1436 hr_dev->caps.qpc_buf_pg_sz + PG_SHIFT_OFFSET); 1437 roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M, 1438 CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S, 1439 qpc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : qpc_hop_num); 1440 1441 roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M, 1442 CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S, 1443 hr_dev->caps.srqc_ba_pg_sz + PG_SHIFT_OFFSET); 1444 roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M, 1445 CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S, 1446 hr_dev->caps.srqc_buf_pg_sz + PG_SHIFT_OFFSET); 1447 roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M, 1448 CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S, 1449 srqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : srqc_hop_num); 1450 1451 roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M, 1452 CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S, 1453 hr_dev->caps.cqc_ba_pg_sz + PG_SHIFT_OFFSET); 1454 roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M, 1455 CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S, 1456 hr_dev->caps.cqc_buf_pg_sz + PG_SHIFT_OFFSET); 1457 roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M, 1458 CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S, 1459 cqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : cqc_hop_num); 1460 1461 roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M, 1462 CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S, 1463 hr_dev->caps.mpt_ba_pg_sz + PG_SHIFT_OFFSET); 1464 roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M, 1465 CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S, 1466 hr_dev->caps.mpt_buf_pg_sz + PG_SHIFT_OFFSET); 1467 roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M, 1468 CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S, 1469 mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num); 1470 1471 roce_set_field(req->vf_sccc_cfg, 1472 CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_M, 1473 CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_S, 1474 hr_dev->caps.sccc_ba_pg_sz + PG_SHIFT_OFFSET); 1475 roce_set_field(req->vf_sccc_cfg, 1476 CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_M, 1477 CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_S, 1478 hr_dev->caps.sccc_buf_pg_sz + PG_SHIFT_OFFSET); 1479 roce_set_field(req->vf_sccc_cfg, 1480 CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_M, 1481 CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_S, 1482 sccc_hop_num == 1483 HNS_ROCE_HOP_NUM_0 ? 0 : sccc_hop_num); 1484 1485 return hns_roce_cmq_send(hr_dev, &desc, 1); 1486 } 1487 1488 static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) 1489 { 1490 struct hns_roce_caps *caps = &hr_dev->caps; 1491 int ret; 1492 1493 ret = hns_roce_cmq_query_hw_info(hr_dev); 1494 if (ret) { 1495 dev_err(hr_dev->dev, "Query hardware version fail, ret = %d.\n", 1496 ret); 1497 return ret; 1498 } 1499 1500 ret = hns_roce_query_fw_ver(hr_dev); 1501 if (ret) { 1502 dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n", 1503 ret); 1504 return ret; 1505 } 1506 1507 ret = hns_roce_config_global_param(hr_dev); 1508 if (ret) { 1509 dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n", 1510 ret); 1511 return ret; 1512 } 1513 1514 /* Get pf resource owned by every pf */ 1515 ret = hns_roce_query_pf_resource(hr_dev); 1516 if (ret) { 1517 dev_err(hr_dev->dev, "Query pf resource fail, ret = %d.\n", 1518 ret); 1519 return ret; 1520 } 1521 1522 if (hr_dev->pci_dev->revision == 0x21) { 1523 ret = hns_roce_query_pf_timer_resource(hr_dev); 1524 if (ret) { 1525 dev_err(hr_dev->dev, 1526 "Query pf timer resource fail, ret = %d.\n", 1527 ret); 1528 return ret; 1529 } 1530 } 1531 1532 ret = hns_roce_alloc_vf_resource(hr_dev); 1533 if (ret) { 1534 dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n", 1535 ret); 1536 return ret; 1537 } 1538 1539 if (hr_dev->pci_dev->revision == 0x21) { 1540 ret = hns_roce_set_vf_switch_param(hr_dev, 0); 1541 if (ret) { 1542 dev_err(hr_dev->dev, 1543 "Set function switch param fail, ret = %d.\n", 1544 ret); 1545 return ret; 1546 } 1547 } 1548 1549 hr_dev->vendor_part_id = hr_dev->pci_dev->device; 1550 hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid); 1551 1552 caps->num_qps = HNS_ROCE_V2_MAX_QP_NUM; 1553 caps->max_wqes = HNS_ROCE_V2_MAX_WQE_NUM; 1554 caps->num_cqs = HNS_ROCE_V2_MAX_CQ_NUM; 1555 caps->num_srqs = HNS_ROCE_V2_MAX_SRQ_NUM; 1556 caps->min_cqes = HNS_ROCE_MIN_CQE_NUM; 1557 caps->max_cqes = HNS_ROCE_V2_MAX_CQE_NUM; 1558 caps->max_srqwqes = HNS_ROCE_V2_MAX_SRQWQE_NUM; 1559 caps->max_sq_sg = HNS_ROCE_V2_MAX_SQ_SGE_NUM; 1560 caps->max_extend_sg = HNS_ROCE_V2_MAX_EXTEND_SGE_NUM; 1561 caps->max_rq_sg = HNS_ROCE_V2_MAX_RQ_SGE_NUM; 1562 caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE; 1563 caps->max_srq_sg = HNS_ROCE_V2_MAX_SRQ_SGE_NUM; 1564 caps->num_uars = HNS_ROCE_V2_UAR_NUM; 1565 caps->phy_num_uars = HNS_ROCE_V2_PHY_UAR_NUM; 1566 caps->num_aeq_vectors = HNS_ROCE_V2_AEQE_VEC_NUM; 1567 caps->num_comp_vectors = HNS_ROCE_V2_COMP_VEC_NUM; 1568 caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM; 1569 caps->num_mtpts = HNS_ROCE_V2_MAX_MTPT_NUM; 1570 caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS; 1571 caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS; 1572 caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS; 1573 caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS; 1574 caps->num_pds = HNS_ROCE_V2_MAX_PD_NUM; 1575 caps->max_qp_init_rdma = HNS_ROCE_V2_MAX_QP_INIT_RDMA; 1576 caps->max_qp_dest_rdma = HNS_ROCE_V2_MAX_QP_DEST_RDMA; 1577 caps->max_sq_desc_sz = HNS_ROCE_V2_MAX_SQ_DESC_SZ; 1578 caps->max_rq_desc_sz = HNS_ROCE_V2_MAX_RQ_DESC_SZ; 1579 caps->max_srq_desc_sz = HNS_ROCE_V2_MAX_SRQ_DESC_SZ; 1580 caps->qpc_entry_sz = HNS_ROCE_V2_QPC_ENTRY_SZ; 1581 caps->irrl_entry_sz = HNS_ROCE_V2_IRRL_ENTRY_SZ; 1582 caps->trrl_entry_sz = HNS_ROCE_V2_TRRL_ENTRY_SZ; 1583 caps->cqc_entry_sz = HNS_ROCE_V2_CQC_ENTRY_SZ; 1584 caps->srqc_entry_sz = HNS_ROCE_V2_SRQC_ENTRY_SZ; 1585 caps->mtpt_entry_sz = HNS_ROCE_V2_MTPT_ENTRY_SZ; 1586 caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ; 1587 caps->idx_entry_sz = 4; 1588 caps->cq_entry_sz = HNS_ROCE_V2_CQE_ENTRY_SIZE; 1589 caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED; 1590 caps->reserved_lkey = 0; 1591 caps->reserved_pds = 0; 1592 caps->reserved_mrws = 1; 1593 caps->reserved_uars = 0; 1594 caps->reserved_cqs = 0; 1595 caps->reserved_srqs = 0; 1596 caps->reserved_qps = HNS_ROCE_V2_RSV_QPS; 1597 1598 caps->qpc_ba_pg_sz = 0; 1599 caps->qpc_buf_pg_sz = 0; 1600 caps->qpc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; 1601 caps->srqc_ba_pg_sz = 0; 1602 caps->srqc_buf_pg_sz = 0; 1603 caps->srqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; 1604 caps->cqc_ba_pg_sz = 0; 1605 caps->cqc_buf_pg_sz = 0; 1606 caps->cqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; 1607 caps->mpt_ba_pg_sz = 0; 1608 caps->mpt_buf_pg_sz = 0; 1609 caps->mpt_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; 1610 caps->pbl_ba_pg_sz = 2; 1611 caps->pbl_buf_pg_sz = 0; 1612 caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM; 1613 caps->mtt_ba_pg_sz = 0; 1614 caps->mtt_buf_pg_sz = 0; 1615 caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM; 1616 caps->wqe_sq_hop_num = 2; 1617 caps->wqe_sge_hop_num = 1; 1618 caps->wqe_rq_hop_num = 2; 1619 caps->cqe_ba_pg_sz = 6; 1620 caps->cqe_buf_pg_sz = 0; 1621 caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM; 1622 caps->srqwqe_ba_pg_sz = 0; 1623 caps->srqwqe_buf_pg_sz = 0; 1624 caps->srqwqe_hop_num = HNS_ROCE_SRQWQE_HOP_NUM; 1625 caps->idx_ba_pg_sz = 0; 1626 caps->idx_buf_pg_sz = 0; 1627 caps->idx_hop_num = HNS_ROCE_IDX_HOP_NUM; 1628 caps->eqe_ba_pg_sz = 0; 1629 caps->eqe_buf_pg_sz = 0; 1630 caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM; 1631 caps->tsq_buf_pg_sz = 0; 1632 caps->chunk_sz = HNS_ROCE_V2_TABLE_CHUNK_SIZE; 1633 1634 caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR | 1635 HNS_ROCE_CAP_FLAG_ROCE_V1_V2 | 1636 HNS_ROCE_CAP_FLAG_RQ_INLINE | 1637 HNS_ROCE_CAP_FLAG_RECORD_DB | 1638 HNS_ROCE_CAP_FLAG_SQ_RECORD_DB; 1639 1640 if (hr_dev->pci_dev->revision == 0x21) 1641 caps->flags |= HNS_ROCE_CAP_FLAG_MW | 1642 HNS_ROCE_CAP_FLAG_FRMR; 1643 1644 caps->pkey_table_len[0] = 1; 1645 caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM; 1646 caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM; 1647 caps->aeqe_depth = HNS_ROCE_V2_ASYNC_EQE_NUM; 1648 caps->local_ca_ack_delay = 0; 1649 caps->max_mtu = IB_MTU_4096; 1650 1651 caps->max_srqs = HNS_ROCE_V2_MAX_SRQ; 1652 caps->max_srq_wrs = HNS_ROCE_V2_MAX_SRQ_WR; 1653 caps->max_srq_sges = HNS_ROCE_V2_MAX_SRQ_SGE; 1654 1655 if (hr_dev->pci_dev->revision == 0x21) { 1656 caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC | 1657 HNS_ROCE_CAP_FLAG_SRQ | 1658 HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL; 1659 1660 caps->num_qpc_timer = HNS_ROCE_V2_MAX_QPC_TIMER_NUM; 1661 caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ; 1662 caps->qpc_timer_ba_pg_sz = 0; 1663 caps->qpc_timer_buf_pg_sz = 0; 1664 caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0; 1665 caps->num_cqc_timer = HNS_ROCE_V2_MAX_CQC_TIMER_NUM; 1666 caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ; 1667 caps->cqc_timer_ba_pg_sz = 0; 1668 caps->cqc_timer_buf_pg_sz = 0; 1669 caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0; 1670 1671 caps->sccc_entry_sz = HNS_ROCE_V2_SCCC_ENTRY_SZ; 1672 caps->sccc_ba_pg_sz = 0; 1673 caps->sccc_buf_pg_sz = 0; 1674 caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM; 1675 } 1676 1677 ret = hns_roce_v2_set_bt(hr_dev); 1678 if (ret) 1679 dev_err(hr_dev->dev, "Configure bt attribute fail, ret = %d.\n", 1680 ret); 1681 1682 return ret; 1683 } 1684 1685 static int hns_roce_config_link_table(struct hns_roce_dev *hr_dev, 1686 enum hns_roce_link_table_type type) 1687 { 1688 struct hns_roce_cmq_desc desc[2]; 1689 struct hns_roce_cfg_llm_a *req_a = 1690 (struct hns_roce_cfg_llm_a *)desc[0].data; 1691 struct hns_roce_cfg_llm_b *req_b = 1692 (struct hns_roce_cfg_llm_b *)desc[1].data; 1693 struct hns_roce_v2_priv *priv = hr_dev->priv; 1694 struct hns_roce_link_table *link_tbl; 1695 struct hns_roce_link_table_entry *entry; 1696 enum hns_roce_opcode_type opcode; 1697 u32 page_num; 1698 int i; 1699 1700 switch (type) { 1701 case TSQ_LINK_TABLE: 1702 link_tbl = &priv->tsq; 1703 opcode = HNS_ROCE_OPC_CFG_EXT_LLM; 1704 break; 1705 case TPQ_LINK_TABLE: 1706 link_tbl = &priv->tpq; 1707 opcode = HNS_ROCE_OPC_CFG_TMOUT_LLM; 1708 break; 1709 default: 1710 return -EINVAL; 1711 } 1712 1713 page_num = link_tbl->npages; 1714 entry = link_tbl->table.buf; 1715 memset(req_a, 0, sizeof(*req_a)); 1716 memset(req_b, 0, sizeof(*req_b)); 1717 1718 for (i = 0; i < 2; i++) { 1719 hns_roce_cmq_setup_basic_desc(&desc[i], opcode, false); 1720 1721 if (i == 0) 1722 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1723 else 1724 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1725 1726 if (i == 0) { 1727 req_a->base_addr_l = link_tbl->table.map & 0xffffffff; 1728 req_a->base_addr_h = (link_tbl->table.map >> 32) & 1729 0xffffffff; 1730 roce_set_field(req_a->depth_pgsz_init_en, 1731 CFG_LLM_QUE_DEPTH_M, 1732 CFG_LLM_QUE_DEPTH_S, 1733 link_tbl->npages); 1734 roce_set_field(req_a->depth_pgsz_init_en, 1735 CFG_LLM_QUE_PGSZ_M, 1736 CFG_LLM_QUE_PGSZ_S, 1737 link_tbl->pg_sz); 1738 req_a->head_ba_l = entry[0].blk_ba0; 1739 req_a->head_ba_h_nxtptr = entry[0].blk_ba1_nxt_ptr; 1740 roce_set_field(req_a->head_ptr, 1741 CFG_LLM_HEAD_PTR_M, 1742 CFG_LLM_HEAD_PTR_S, 0); 1743 } else { 1744 req_b->tail_ba_l = entry[page_num - 1].blk_ba0; 1745 roce_set_field(req_b->tail_ba_h, 1746 CFG_LLM_TAIL_BA_H_M, 1747 CFG_LLM_TAIL_BA_H_S, 1748 entry[page_num - 1].blk_ba1_nxt_ptr & 1749 HNS_ROCE_LINK_TABLE_BA1_M); 1750 roce_set_field(req_b->tail_ptr, 1751 CFG_LLM_TAIL_PTR_M, 1752 CFG_LLM_TAIL_PTR_S, 1753 (entry[page_num - 2].blk_ba1_nxt_ptr & 1754 HNS_ROCE_LINK_TABLE_NXT_PTR_M) >> 1755 HNS_ROCE_LINK_TABLE_NXT_PTR_S); 1756 } 1757 } 1758 roce_set_field(req_a->depth_pgsz_init_en, 1759 CFG_LLM_INIT_EN_M, CFG_LLM_INIT_EN_S, 1); 1760 1761 return hns_roce_cmq_send(hr_dev, desc, 2); 1762 } 1763 1764 static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev, 1765 enum hns_roce_link_table_type type) 1766 { 1767 struct hns_roce_v2_priv *priv = hr_dev->priv; 1768 struct hns_roce_link_table *link_tbl; 1769 struct hns_roce_link_table_entry *entry; 1770 struct device *dev = hr_dev->dev; 1771 u32 buf_chk_sz; 1772 dma_addr_t t; 1773 int func_num = 1; 1774 int pg_num_a; 1775 int pg_num_b; 1776 int pg_num; 1777 int size; 1778 int i; 1779 1780 switch (type) { 1781 case TSQ_LINK_TABLE: 1782 link_tbl = &priv->tsq; 1783 buf_chk_sz = 1 << (hr_dev->caps.tsq_buf_pg_sz + PAGE_SHIFT); 1784 pg_num_a = hr_dev->caps.num_qps * 8 / buf_chk_sz; 1785 pg_num_b = hr_dev->caps.sl_num * 4 + 2; 1786 break; 1787 case TPQ_LINK_TABLE: 1788 link_tbl = &priv->tpq; 1789 buf_chk_sz = 1 << (hr_dev->caps.tpq_buf_pg_sz + PAGE_SHIFT); 1790 pg_num_a = hr_dev->caps.num_cqs * 4 / buf_chk_sz; 1791 pg_num_b = 2 * 4 * func_num + 2; 1792 break; 1793 default: 1794 return -EINVAL; 1795 } 1796 1797 pg_num = max(pg_num_a, pg_num_b); 1798 size = pg_num * sizeof(struct hns_roce_link_table_entry); 1799 1800 link_tbl->table.buf = dma_alloc_coherent(dev, size, 1801 &link_tbl->table.map, 1802 GFP_KERNEL); 1803 if (!link_tbl->table.buf) 1804 goto out; 1805 1806 link_tbl->pg_list = kcalloc(pg_num, sizeof(*link_tbl->pg_list), 1807 GFP_KERNEL); 1808 if (!link_tbl->pg_list) 1809 goto err_kcalloc_failed; 1810 1811 entry = link_tbl->table.buf; 1812 for (i = 0; i < pg_num; ++i) { 1813 link_tbl->pg_list[i].buf = dma_alloc_coherent(dev, buf_chk_sz, 1814 &t, GFP_KERNEL); 1815 if (!link_tbl->pg_list[i].buf) 1816 goto err_alloc_buf_failed; 1817 1818 link_tbl->pg_list[i].map = t; 1819 1820 entry[i].blk_ba0 = (t >> 12) & 0xffffffff; 1821 roce_set_field(entry[i].blk_ba1_nxt_ptr, 1822 HNS_ROCE_LINK_TABLE_BA1_M, 1823 HNS_ROCE_LINK_TABLE_BA1_S, 1824 t >> 44); 1825 1826 if (i < (pg_num - 1)) 1827 roce_set_field(entry[i].blk_ba1_nxt_ptr, 1828 HNS_ROCE_LINK_TABLE_NXT_PTR_M, 1829 HNS_ROCE_LINK_TABLE_NXT_PTR_S, 1830 i + 1); 1831 } 1832 link_tbl->npages = pg_num; 1833 link_tbl->pg_sz = buf_chk_sz; 1834 1835 return hns_roce_config_link_table(hr_dev, type); 1836 1837 err_alloc_buf_failed: 1838 for (i -= 1; i >= 0; i--) 1839 dma_free_coherent(dev, buf_chk_sz, 1840 link_tbl->pg_list[i].buf, 1841 link_tbl->pg_list[i].map); 1842 kfree(link_tbl->pg_list); 1843 1844 err_kcalloc_failed: 1845 dma_free_coherent(dev, size, link_tbl->table.buf, 1846 link_tbl->table.map); 1847 1848 out: 1849 return -ENOMEM; 1850 } 1851 1852 static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev, 1853 struct hns_roce_link_table *link_tbl) 1854 { 1855 struct device *dev = hr_dev->dev; 1856 int size; 1857 int i; 1858 1859 size = link_tbl->npages * sizeof(struct hns_roce_link_table_entry); 1860 1861 for (i = 0; i < link_tbl->npages; ++i) 1862 if (link_tbl->pg_list[i].buf) 1863 dma_free_coherent(dev, link_tbl->pg_sz, 1864 link_tbl->pg_list[i].buf, 1865 link_tbl->pg_list[i].map); 1866 kfree(link_tbl->pg_list); 1867 1868 dma_free_coherent(dev, size, link_tbl->table.buf, 1869 link_tbl->table.map); 1870 } 1871 1872 static int hns_roce_v2_init(struct hns_roce_dev *hr_dev) 1873 { 1874 struct hns_roce_v2_priv *priv = hr_dev->priv; 1875 int qpc_count, cqc_count; 1876 int ret, i; 1877 1878 /* TSQ includes SQ doorbell and ack doorbell */ 1879 ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE); 1880 if (ret) { 1881 dev_err(hr_dev->dev, "TSQ init failed, ret = %d.\n", ret); 1882 return ret; 1883 } 1884 1885 ret = hns_roce_init_link_table(hr_dev, TPQ_LINK_TABLE); 1886 if (ret) { 1887 dev_err(hr_dev->dev, "TPQ init failed, ret = %d.\n", ret); 1888 goto err_tpq_init_failed; 1889 } 1890 1891 /* Alloc memory for QPC Timer buffer space chunk*/ 1892 for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num; 1893 qpc_count++) { 1894 ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table, 1895 qpc_count); 1896 if (ret) { 1897 dev_err(hr_dev->dev, "QPC Timer get failed\n"); 1898 goto err_qpc_timer_failed; 1899 } 1900 } 1901 1902 /* Alloc memory for CQC Timer buffer space chunk*/ 1903 for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num; 1904 cqc_count++) { 1905 ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table, 1906 cqc_count); 1907 if (ret) { 1908 dev_err(hr_dev->dev, "CQC Timer get failed\n"); 1909 goto err_cqc_timer_failed; 1910 } 1911 } 1912 1913 return 0; 1914 1915 err_cqc_timer_failed: 1916 for (i = 0; i < cqc_count; i++) 1917 hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i); 1918 1919 err_qpc_timer_failed: 1920 for (i = 0; i < qpc_count; i++) 1921 hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i); 1922 1923 hns_roce_free_link_table(hr_dev, &priv->tpq); 1924 1925 err_tpq_init_failed: 1926 hns_roce_free_link_table(hr_dev, &priv->tsq); 1927 1928 return ret; 1929 } 1930 1931 static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev) 1932 { 1933 struct hns_roce_v2_priv *priv = hr_dev->priv; 1934 1935 if (hr_dev->pci_dev->revision == 0x21) 1936 hns_roce_function_clear(hr_dev); 1937 1938 hns_roce_free_link_table(hr_dev, &priv->tpq); 1939 hns_roce_free_link_table(hr_dev, &priv->tsq); 1940 } 1941 1942 static int hns_roce_query_mbox_status(struct hns_roce_dev *hr_dev) 1943 { 1944 struct hns_roce_cmq_desc desc; 1945 struct hns_roce_mbox_status *mb_st = 1946 (struct hns_roce_mbox_status *)desc.data; 1947 enum hns_roce_cmd_return_status status; 1948 1949 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST, true); 1950 1951 status = hns_roce_cmq_send(hr_dev, &desc, 1); 1952 if (status) 1953 return status; 1954 1955 return cpu_to_le32(mb_st->mb_status_hw_run); 1956 } 1957 1958 static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev) 1959 { 1960 u32 status = hns_roce_query_mbox_status(hr_dev); 1961 1962 return status >> HNS_ROCE_HW_RUN_BIT_SHIFT; 1963 } 1964 1965 static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev) 1966 { 1967 u32 status = hns_roce_query_mbox_status(hr_dev); 1968 1969 return status & HNS_ROCE_HW_MB_STATUS_MASK; 1970 } 1971 1972 static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, u64 in_param, 1973 u64 out_param, u32 in_modifier, u8 op_modifier, 1974 u16 op, u16 token, int event) 1975 { 1976 struct hns_roce_cmq_desc desc; 1977 struct hns_roce_post_mbox *mb = (struct hns_roce_post_mbox *)desc.data; 1978 1979 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false); 1980 1981 mb->in_param_l = cpu_to_le64(in_param); 1982 mb->in_param_h = cpu_to_le64(in_param) >> 32; 1983 mb->out_param_l = cpu_to_le64(out_param); 1984 mb->out_param_h = cpu_to_le64(out_param) >> 32; 1985 mb->cmd_tag = cpu_to_le32(in_modifier << 8 | op); 1986 mb->token_event_en = cpu_to_le32(event << 16 | token); 1987 1988 return hns_roce_cmq_send(hr_dev, &desc, 1); 1989 } 1990 1991 static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param, 1992 u64 out_param, u32 in_modifier, u8 op_modifier, 1993 u16 op, u16 token, int event) 1994 { 1995 struct device *dev = hr_dev->dev; 1996 unsigned long end; 1997 int ret; 1998 1999 end = msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS) + jiffies; 2000 while (hns_roce_v2_cmd_pending(hr_dev)) { 2001 if (time_after(jiffies, end)) { 2002 dev_dbg(dev, "jiffies=%d end=%d\n", (int)jiffies, 2003 (int)end); 2004 return -EAGAIN; 2005 } 2006 cond_resched(); 2007 } 2008 2009 ret = hns_roce_mbox_post(hr_dev, in_param, out_param, in_modifier, 2010 op_modifier, op, token, event); 2011 if (ret) 2012 dev_err(dev, "Post mailbox fail(%d)\n", ret); 2013 2014 return ret; 2015 } 2016 2017 static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev, 2018 unsigned long timeout) 2019 { 2020 struct device *dev = hr_dev->dev; 2021 unsigned long end; 2022 u32 status; 2023 2024 end = msecs_to_jiffies(timeout) + jiffies; 2025 while (hns_roce_v2_cmd_pending(hr_dev) && time_before(jiffies, end)) 2026 cond_resched(); 2027 2028 if (hns_roce_v2_cmd_pending(hr_dev)) { 2029 dev_err(dev, "[cmd_poll]hw run cmd TIMEDOUT!\n"); 2030 return -ETIMEDOUT; 2031 } 2032 2033 status = hns_roce_v2_cmd_complete(hr_dev); 2034 if (status != 0x1) { 2035 if (status == CMD_RST_PRC_EBUSY) 2036 return status; 2037 2038 dev_err(dev, "mailbox status 0x%x!\n", status); 2039 return -EBUSY; 2040 } 2041 2042 return 0; 2043 } 2044 2045 static int hns_roce_config_sgid_table(struct hns_roce_dev *hr_dev, 2046 int gid_index, const union ib_gid *gid, 2047 enum hns_roce_sgid_type sgid_type) 2048 { 2049 struct hns_roce_cmq_desc desc; 2050 struct hns_roce_cfg_sgid_tb *sgid_tb = 2051 (struct hns_roce_cfg_sgid_tb *)desc.data; 2052 u32 *p; 2053 2054 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false); 2055 2056 roce_set_field(sgid_tb->table_idx_rsv, 2057 CFG_SGID_TB_TABLE_IDX_M, 2058 CFG_SGID_TB_TABLE_IDX_S, gid_index); 2059 roce_set_field(sgid_tb->vf_sgid_type_rsv, 2060 CFG_SGID_TB_VF_SGID_TYPE_M, 2061 CFG_SGID_TB_VF_SGID_TYPE_S, sgid_type); 2062 2063 p = (u32 *)&gid->raw[0]; 2064 sgid_tb->vf_sgid_l = cpu_to_le32(*p); 2065 2066 p = (u32 *)&gid->raw[4]; 2067 sgid_tb->vf_sgid_ml = cpu_to_le32(*p); 2068 2069 p = (u32 *)&gid->raw[8]; 2070 sgid_tb->vf_sgid_mh = cpu_to_le32(*p); 2071 2072 p = (u32 *)&gid->raw[0xc]; 2073 sgid_tb->vf_sgid_h = cpu_to_le32(*p); 2074 2075 return hns_roce_cmq_send(hr_dev, &desc, 1); 2076 } 2077 2078 static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port, 2079 int gid_index, const union ib_gid *gid, 2080 const struct ib_gid_attr *attr) 2081 { 2082 enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1; 2083 int ret; 2084 2085 if (!gid || !attr) 2086 return -EINVAL; 2087 2088 if (attr->gid_type == IB_GID_TYPE_ROCE) 2089 sgid_type = GID_TYPE_FLAG_ROCE_V1; 2090 2091 if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { 2092 if (ipv6_addr_v4mapped((void *)gid)) 2093 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4; 2094 else 2095 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6; 2096 } 2097 2098 ret = hns_roce_config_sgid_table(hr_dev, gid_index, gid, sgid_type); 2099 if (ret) 2100 dev_err(hr_dev->dev, "Configure sgid table failed(%d)!\n", ret); 2101 2102 return ret; 2103 } 2104 2105 static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, 2106 u8 *addr) 2107 { 2108 struct hns_roce_cmq_desc desc; 2109 struct hns_roce_cfg_smac_tb *smac_tb = 2110 (struct hns_roce_cfg_smac_tb *)desc.data; 2111 u16 reg_smac_h; 2112 u32 reg_smac_l; 2113 2114 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SMAC_TB, false); 2115 2116 reg_smac_l = *(u32 *)(&addr[0]); 2117 reg_smac_h = *(u16 *)(&addr[4]); 2118 2119 memset(smac_tb, 0, sizeof(*smac_tb)); 2120 roce_set_field(smac_tb->tb_idx_rsv, 2121 CFG_SMAC_TB_IDX_M, 2122 CFG_SMAC_TB_IDX_S, phy_port); 2123 roce_set_field(smac_tb->vf_smac_h_rsv, 2124 CFG_SMAC_TB_VF_SMAC_H_M, 2125 CFG_SMAC_TB_VF_SMAC_H_S, reg_smac_h); 2126 smac_tb->vf_smac_l = reg_smac_l; 2127 2128 return hns_roce_cmq_send(hr_dev, &desc, 1); 2129 } 2130 2131 static int set_mtpt_pbl(struct hns_roce_v2_mpt_entry *mpt_entry, 2132 struct hns_roce_mr *mr) 2133 { 2134 struct sg_dma_page_iter sg_iter; 2135 u64 page_addr; 2136 u64 *pages; 2137 int i; 2138 2139 mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size); 2140 mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3)); 2141 roce_set_field(mpt_entry->byte_48_mode_ba, 2142 V2_MPT_BYTE_48_PBL_BA_H_M, V2_MPT_BYTE_48_PBL_BA_H_S, 2143 upper_32_bits(mr->pbl_ba >> 3)); 2144 2145 pages = (u64 *)__get_free_page(GFP_KERNEL); 2146 if (!pages) 2147 return -ENOMEM; 2148 2149 i = 0; 2150 for_each_sg_dma_page(mr->umem->sg_head.sgl, &sg_iter, mr->umem->nmap, 0) { 2151 page_addr = sg_page_iter_dma_address(&sg_iter); 2152 pages[i] = page_addr >> 6; 2153 2154 /* Record the first 2 entry directly to MTPT table */ 2155 if (i >= HNS_ROCE_V2_MAX_INNER_MTPT_NUM - 1) 2156 goto found; 2157 i++; 2158 } 2159 found: 2160 mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0])); 2161 roce_set_field(mpt_entry->byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M, 2162 V2_MPT_BYTE_56_PA0_H_S, upper_32_bits(pages[0])); 2163 2164 mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1])); 2165 roce_set_field(mpt_entry->byte_64_buf_pa1, V2_MPT_BYTE_64_PA1_H_M, 2166 V2_MPT_BYTE_64_PA1_H_S, upper_32_bits(pages[1])); 2167 roce_set_field(mpt_entry->byte_64_buf_pa1, 2168 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M, 2169 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S, 2170 mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET); 2171 2172 free_page((unsigned long)pages); 2173 2174 return 0; 2175 } 2176 2177 static int hns_roce_v2_write_mtpt(void *mb_buf, struct hns_roce_mr *mr, 2178 unsigned long mtpt_idx) 2179 { 2180 struct hns_roce_v2_mpt_entry *mpt_entry; 2181 int ret; 2182 2183 mpt_entry = mb_buf; 2184 memset(mpt_entry, 0, sizeof(*mpt_entry)); 2185 2186 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M, 2187 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID); 2188 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M, 2189 V2_MPT_BYTE_4_PBL_HOP_NUM_S, mr->pbl_hop_num == 2190 HNS_ROCE_HOP_NUM_0 ? 0 : mr->pbl_hop_num); 2191 roce_set_field(mpt_entry->byte_4_pd_hop_st, 2192 V2_MPT_BYTE_4_PBL_BA_PG_SZ_M, 2193 V2_MPT_BYTE_4_PBL_BA_PG_SZ_S, 2194 mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET); 2195 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M, 2196 V2_MPT_BYTE_4_PD_S, mr->pd); 2197 2198 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 0); 2199 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 0); 2200 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1); 2201 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_BIND_EN_S, 2202 (mr->access & IB_ACCESS_MW_BIND ? 1 : 0)); 2203 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_ATOMIC_EN_S, 2204 mr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0); 2205 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S, 2206 (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0)); 2207 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S, 2208 (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0)); 2209 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S, 2210 (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0)); 2211 2212 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 2213 mr->type == MR_TYPE_MR ? 0 : 1); 2214 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_INNER_PA_VLD_S, 2215 1); 2216 2217 mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size)); 2218 mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size)); 2219 mpt_entry->lkey = cpu_to_le32(mr->key); 2220 mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova)); 2221 mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova)); 2222 2223 if (mr->type == MR_TYPE_DMA) 2224 return 0; 2225 2226 ret = set_mtpt_pbl(mpt_entry, mr); 2227 2228 return ret; 2229 } 2230 2231 static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev, 2232 struct hns_roce_mr *mr, int flags, 2233 u32 pdn, int mr_access_flags, u64 iova, 2234 u64 size, void *mb_buf) 2235 { 2236 struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf; 2237 int ret = 0; 2238 2239 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M, 2240 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID); 2241 2242 if (flags & IB_MR_REREG_PD) { 2243 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M, 2244 V2_MPT_BYTE_4_PD_S, pdn); 2245 mr->pd = pdn; 2246 } 2247 2248 if (flags & IB_MR_REREG_ACCESS) { 2249 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, 2250 V2_MPT_BYTE_8_BIND_EN_S, 2251 (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0)); 2252 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, 2253 V2_MPT_BYTE_8_ATOMIC_EN_S, 2254 mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0); 2255 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S, 2256 mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0); 2257 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S, 2258 mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0); 2259 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S, 2260 mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0); 2261 } 2262 2263 if (flags & IB_MR_REREG_TRANS) { 2264 mpt_entry->va_l = cpu_to_le32(lower_32_bits(iova)); 2265 mpt_entry->va_h = cpu_to_le32(upper_32_bits(iova)); 2266 mpt_entry->len_l = cpu_to_le32(lower_32_bits(size)); 2267 mpt_entry->len_h = cpu_to_le32(upper_32_bits(size)); 2268 2269 mr->iova = iova; 2270 mr->size = size; 2271 2272 ret = set_mtpt_pbl(mpt_entry, mr); 2273 } 2274 2275 return ret; 2276 } 2277 2278 static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr) 2279 { 2280 struct hns_roce_v2_mpt_entry *mpt_entry; 2281 2282 mpt_entry = mb_buf; 2283 memset(mpt_entry, 0, sizeof(*mpt_entry)); 2284 2285 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M, 2286 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE); 2287 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M, 2288 V2_MPT_BYTE_4_PBL_HOP_NUM_S, 1); 2289 roce_set_field(mpt_entry->byte_4_pd_hop_st, 2290 V2_MPT_BYTE_4_PBL_BA_PG_SZ_M, 2291 V2_MPT_BYTE_4_PBL_BA_PG_SZ_S, 2292 mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET); 2293 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M, 2294 V2_MPT_BYTE_4_PD_S, mr->pd); 2295 2296 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 1); 2297 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1); 2298 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1); 2299 2300 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_FRE_S, 1); 2301 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0); 2302 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 0); 2303 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1); 2304 2305 mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size); 2306 2307 mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3)); 2308 roce_set_field(mpt_entry->byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M, 2309 V2_MPT_BYTE_48_PBL_BA_H_S, 2310 upper_32_bits(mr->pbl_ba >> 3)); 2311 2312 roce_set_field(mpt_entry->byte_64_buf_pa1, 2313 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M, 2314 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S, 2315 mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET); 2316 2317 return 0; 2318 } 2319 2320 static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw) 2321 { 2322 struct hns_roce_v2_mpt_entry *mpt_entry; 2323 2324 mpt_entry = mb_buf; 2325 memset(mpt_entry, 0, sizeof(*mpt_entry)); 2326 2327 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M, 2328 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE); 2329 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M, 2330 V2_MPT_BYTE_4_PD_S, mw->pdn); 2331 roce_set_field(mpt_entry->byte_4_pd_hop_st, 2332 V2_MPT_BYTE_4_PBL_HOP_NUM_M, 2333 V2_MPT_BYTE_4_PBL_HOP_NUM_S, 2334 mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ? 2335 0 : mw->pbl_hop_num); 2336 roce_set_field(mpt_entry->byte_4_pd_hop_st, 2337 V2_MPT_BYTE_4_PBL_BA_PG_SZ_M, 2338 V2_MPT_BYTE_4_PBL_BA_PG_SZ_S, 2339 mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET); 2340 2341 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1); 2342 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1); 2343 2344 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0); 2345 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 1); 2346 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1); 2347 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BQP_S, 2348 mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1); 2349 2350 roce_set_field(mpt_entry->byte_64_buf_pa1, 2351 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M, 2352 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S, 2353 mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET); 2354 2355 mpt_entry->lkey = cpu_to_le32(mw->rkey); 2356 2357 return 0; 2358 } 2359 2360 static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n) 2361 { 2362 return hns_roce_buf_offset(&hr_cq->hr_buf.hr_buf, 2363 n * HNS_ROCE_V2_CQE_ENTRY_SIZE); 2364 } 2365 2366 static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, int n) 2367 { 2368 struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe); 2369 2370 /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */ 2371 return (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_OWNER_S) ^ 2372 !!(n & (hr_cq->ib_cq.cqe + 1))) ? cqe : NULL; 2373 } 2374 2375 static struct hns_roce_v2_cqe *next_cqe_sw_v2(struct hns_roce_cq *hr_cq) 2376 { 2377 return get_sw_cqe_v2(hr_cq, hr_cq->cons_index); 2378 } 2379 2380 static void *get_srq_wqe(struct hns_roce_srq *srq, int n) 2381 { 2382 return hns_roce_buf_offset(&srq->buf, n << srq->wqe_shift); 2383 } 2384 2385 static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index) 2386 { 2387 /* always called with interrupts disabled. */ 2388 spin_lock(&srq->lock); 2389 2390 bitmap_clear(srq->idx_que.bitmap, wqe_index, 1); 2391 srq->tail++; 2392 2393 spin_unlock(&srq->lock); 2394 } 2395 2396 static void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index) 2397 { 2398 *hr_cq->set_ci_db = cons_index & 0xffffff; 2399 } 2400 2401 static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, 2402 struct hns_roce_srq *srq) 2403 { 2404 struct hns_roce_v2_cqe *cqe, *dest; 2405 u32 prod_index; 2406 int nfreed = 0; 2407 int wqe_index; 2408 u8 owner_bit; 2409 2410 for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index); 2411 ++prod_index) { 2412 if (prod_index == hr_cq->cons_index + hr_cq->ib_cq.cqe) 2413 break; 2414 } 2415 2416 /* 2417 * Now backwards through the CQ, removing CQ entries 2418 * that match our QP by overwriting them with next entries. 2419 */ 2420 while ((int) --prod_index - (int) hr_cq->cons_index >= 0) { 2421 cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe); 2422 if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M, 2423 V2_CQE_BYTE_16_LCL_QPN_S) & 2424 HNS_ROCE_V2_CQE_QPN_MASK) == qpn) { 2425 if (srq && 2426 roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S)) { 2427 wqe_index = roce_get_field(cqe->byte_4, 2428 V2_CQE_BYTE_4_WQE_INDX_M, 2429 V2_CQE_BYTE_4_WQE_INDX_S); 2430 hns_roce_free_srq_wqe(srq, wqe_index); 2431 } 2432 ++nfreed; 2433 } else if (nfreed) { 2434 dest = get_cqe_v2(hr_cq, (prod_index + nfreed) & 2435 hr_cq->ib_cq.cqe); 2436 owner_bit = roce_get_bit(dest->byte_4, 2437 V2_CQE_BYTE_4_OWNER_S); 2438 memcpy(dest, cqe, sizeof(*cqe)); 2439 roce_set_bit(dest->byte_4, V2_CQE_BYTE_4_OWNER_S, 2440 owner_bit); 2441 } 2442 } 2443 2444 if (nfreed) { 2445 hr_cq->cons_index += nfreed; 2446 /* 2447 * Make sure update of buffer contents is done before 2448 * updating consumer index. 2449 */ 2450 wmb(); 2451 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index); 2452 } 2453 } 2454 2455 static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, 2456 struct hns_roce_srq *srq) 2457 { 2458 spin_lock_irq(&hr_cq->lock); 2459 __hns_roce_v2_cq_clean(hr_cq, qpn, srq); 2460 spin_unlock_irq(&hr_cq->lock); 2461 } 2462 2463 static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, 2464 struct hns_roce_cq *hr_cq, void *mb_buf, 2465 u64 *mtts, dma_addr_t dma_handle, int nent, 2466 u32 vector) 2467 { 2468 struct hns_roce_v2_cq_context *cq_context; 2469 2470 cq_context = mb_buf; 2471 memset(cq_context, 0, sizeof(*cq_context)); 2472 2473 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CQ_ST_M, 2474 V2_CQC_BYTE_4_CQ_ST_S, V2_CQ_STATE_VALID); 2475 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M, 2476 V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE); 2477 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M, 2478 V2_CQC_BYTE_4_SHIFT_S, ilog2((unsigned int)nent)); 2479 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M, 2480 V2_CQC_BYTE_4_CEQN_S, vector); 2481 cq_context->byte_4_pg_ceqn = cpu_to_le32(cq_context->byte_4_pg_ceqn); 2482 2483 roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQN_M, 2484 V2_CQC_BYTE_8_CQN_S, hr_cq->cqn); 2485 2486 cq_context->cqe_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT); 2487 cq_context->cqe_cur_blk_addr = 2488 cpu_to_le32(cq_context->cqe_cur_blk_addr); 2489 2490 roce_set_field(cq_context->byte_16_hop_addr, 2491 V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M, 2492 V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S, 2493 cpu_to_le32((mtts[0]) >> (32 + PAGE_ADDR_SHIFT))); 2494 roce_set_field(cq_context->byte_16_hop_addr, 2495 V2_CQC_BYTE_16_CQE_HOP_NUM_M, 2496 V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev->caps.cqe_hop_num == 2497 HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num); 2498 2499 cq_context->cqe_nxt_blk_addr = (u32)(mtts[1] >> PAGE_ADDR_SHIFT); 2500 roce_set_field(cq_context->byte_24_pgsz_addr, 2501 V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M, 2502 V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S, 2503 cpu_to_le32((mtts[1]) >> (32 + PAGE_ADDR_SHIFT))); 2504 roce_set_field(cq_context->byte_24_pgsz_addr, 2505 V2_CQC_BYTE_24_CQE_BA_PG_SZ_M, 2506 V2_CQC_BYTE_24_CQE_BA_PG_SZ_S, 2507 hr_dev->caps.cqe_ba_pg_sz + PG_SHIFT_OFFSET); 2508 roce_set_field(cq_context->byte_24_pgsz_addr, 2509 V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M, 2510 V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S, 2511 hr_dev->caps.cqe_buf_pg_sz + PG_SHIFT_OFFSET); 2512 2513 cq_context->cqe_ba = (u32)(dma_handle >> 3); 2514 2515 roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M, 2516 V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3))); 2517 2518 if (hr_cq->db_en) 2519 roce_set_bit(cq_context->byte_44_db_record, 2520 V2_CQC_BYTE_44_DB_RECORD_EN_S, 1); 2521 2522 roce_set_field(cq_context->byte_44_db_record, 2523 V2_CQC_BYTE_44_DB_RECORD_ADDR_M, 2524 V2_CQC_BYTE_44_DB_RECORD_ADDR_S, 2525 ((u32)hr_cq->db.dma) >> 1); 2526 cq_context->db_record_addr = hr_cq->db.dma >> 32; 2527 2528 roce_set_field(cq_context->byte_56_cqe_period_maxcnt, 2529 V2_CQC_BYTE_56_CQ_MAX_CNT_M, 2530 V2_CQC_BYTE_56_CQ_MAX_CNT_S, 2531 HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM); 2532 roce_set_field(cq_context->byte_56_cqe_period_maxcnt, 2533 V2_CQC_BYTE_56_CQ_PERIOD_M, 2534 V2_CQC_BYTE_56_CQ_PERIOD_S, 2535 HNS_ROCE_V2_CQ_DEFAULT_INTERVAL); 2536 } 2537 2538 static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq, 2539 enum ib_cq_notify_flags flags) 2540 { 2541 struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); 2542 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); 2543 u32 notification_flag; 2544 u32 doorbell[2]; 2545 2546 doorbell[0] = 0; 2547 doorbell[1] = 0; 2548 2549 notification_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? 2550 V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL; 2551 /* 2552 * flags = 0; Notification Flag = 1, next 2553 * flags = 1; Notification Flag = 0, solocited 2554 */ 2555 roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_TAG_M, V2_DB_BYTE_4_TAG_S, 2556 hr_cq->cqn); 2557 roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S, 2558 HNS_ROCE_V2_CQ_DB_NTR); 2559 roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M, 2560 V2_CQ_DB_PARAMETER_CONS_IDX_S, 2561 hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1)); 2562 roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M, 2563 V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq->arm_sn & 0x3); 2564 roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S, 2565 notification_flag); 2566 2567 hns_roce_write64(hr_dev, doorbell, hr_cq->cq_db_l); 2568 2569 return 0; 2570 } 2571 2572 static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe, 2573 struct hns_roce_qp **cur_qp, 2574 struct ib_wc *wc) 2575 { 2576 struct hns_roce_rinl_sge *sge_list; 2577 u32 wr_num, wr_cnt, sge_num; 2578 u32 sge_cnt, data_len, size; 2579 void *wqe_buf; 2580 2581 wr_num = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_WQE_INDX_M, 2582 V2_CQE_BYTE_4_WQE_INDX_S) & 0xffff; 2583 wr_cnt = wr_num & ((*cur_qp)->rq.wqe_cnt - 1); 2584 2585 sge_list = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sg_list; 2586 sge_num = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sge_cnt; 2587 wqe_buf = get_recv_wqe(*cur_qp, wr_cnt); 2588 data_len = wc->byte_len; 2589 2590 for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) { 2591 size = min(sge_list[sge_cnt].len, data_len); 2592 memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size); 2593 2594 data_len -= size; 2595 wqe_buf += size; 2596 } 2597 2598 if (data_len) { 2599 wc->status = IB_WC_LOC_LEN_ERR; 2600 return -EAGAIN; 2601 } 2602 2603 return 0; 2604 } 2605 2606 static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, 2607 struct hns_roce_qp **cur_qp, struct ib_wc *wc) 2608 { 2609 struct hns_roce_srq *srq = NULL; 2610 struct hns_roce_dev *hr_dev; 2611 struct hns_roce_v2_cqe *cqe; 2612 struct hns_roce_qp *hr_qp; 2613 struct hns_roce_wq *wq; 2614 struct ib_qp_attr attr; 2615 int attr_mask; 2616 int is_send; 2617 u16 wqe_ctr; 2618 u32 opcode; 2619 u32 status; 2620 int qpn; 2621 int ret; 2622 2623 /* Find cqe according to consumer index */ 2624 cqe = next_cqe_sw_v2(hr_cq); 2625 if (!cqe) 2626 return -EAGAIN; 2627 2628 ++hr_cq->cons_index; 2629 /* Memory barrier */ 2630 rmb(); 2631 2632 /* 0->SQ, 1->RQ */ 2633 is_send = !roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S); 2634 2635 qpn = roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M, 2636 V2_CQE_BYTE_16_LCL_QPN_S); 2637 2638 if (!*cur_qp || (qpn & HNS_ROCE_V2_CQE_QPN_MASK) != (*cur_qp)->qpn) { 2639 hr_dev = to_hr_dev(hr_cq->ib_cq.device); 2640 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn); 2641 if (unlikely(!hr_qp)) { 2642 dev_err(hr_dev->dev, "CQ %06lx with entry for unknown QPN %06x\n", 2643 hr_cq->cqn, (qpn & HNS_ROCE_V2_CQE_QPN_MASK)); 2644 return -EINVAL; 2645 } 2646 *cur_qp = hr_qp; 2647 } 2648 2649 wc->qp = &(*cur_qp)->ibqp; 2650 wc->vendor_err = 0; 2651 2652 if (is_send) { 2653 wq = &(*cur_qp)->sq; 2654 if ((*cur_qp)->sq_signal_bits) { 2655 /* 2656 * If sg_signal_bit is 1, 2657 * firstly tail pointer updated to wqe 2658 * which current cqe correspond to 2659 */ 2660 wqe_ctr = (u16)roce_get_field(cqe->byte_4, 2661 V2_CQE_BYTE_4_WQE_INDX_M, 2662 V2_CQE_BYTE_4_WQE_INDX_S); 2663 wq->tail += (wqe_ctr - (u16)wq->tail) & 2664 (wq->wqe_cnt - 1); 2665 } 2666 2667 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; 2668 ++wq->tail; 2669 } else if ((*cur_qp)->ibqp.srq) { 2670 srq = to_hr_srq((*cur_qp)->ibqp.srq); 2671 wqe_ctr = le16_to_cpu(roce_get_field(cqe->byte_4, 2672 V2_CQE_BYTE_4_WQE_INDX_M, 2673 V2_CQE_BYTE_4_WQE_INDX_S)); 2674 wc->wr_id = srq->wrid[wqe_ctr]; 2675 hns_roce_free_srq_wqe(srq, wqe_ctr); 2676 } else { 2677 /* Update tail pointer, record wr_id */ 2678 wq = &(*cur_qp)->rq; 2679 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; 2680 ++wq->tail; 2681 } 2682 2683 status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M, 2684 V2_CQE_BYTE_4_STATUS_S); 2685 switch (status & HNS_ROCE_V2_CQE_STATUS_MASK) { 2686 case HNS_ROCE_CQE_V2_SUCCESS: 2687 wc->status = IB_WC_SUCCESS; 2688 break; 2689 case HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR: 2690 wc->status = IB_WC_LOC_LEN_ERR; 2691 break; 2692 case HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR: 2693 wc->status = IB_WC_LOC_QP_OP_ERR; 2694 break; 2695 case HNS_ROCE_CQE_V2_LOCAL_PROT_ERR: 2696 wc->status = IB_WC_LOC_PROT_ERR; 2697 break; 2698 case HNS_ROCE_CQE_V2_WR_FLUSH_ERR: 2699 wc->status = IB_WC_WR_FLUSH_ERR; 2700 break; 2701 case HNS_ROCE_CQE_V2_MW_BIND_ERR: 2702 wc->status = IB_WC_MW_BIND_ERR; 2703 break; 2704 case HNS_ROCE_CQE_V2_BAD_RESP_ERR: 2705 wc->status = IB_WC_BAD_RESP_ERR; 2706 break; 2707 case HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR: 2708 wc->status = IB_WC_LOC_ACCESS_ERR; 2709 break; 2710 case HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR: 2711 wc->status = IB_WC_REM_INV_REQ_ERR; 2712 break; 2713 case HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR: 2714 wc->status = IB_WC_REM_ACCESS_ERR; 2715 break; 2716 case HNS_ROCE_CQE_V2_REMOTE_OP_ERR: 2717 wc->status = IB_WC_REM_OP_ERR; 2718 break; 2719 case HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR: 2720 wc->status = IB_WC_RETRY_EXC_ERR; 2721 break; 2722 case HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR: 2723 wc->status = IB_WC_RNR_RETRY_EXC_ERR; 2724 break; 2725 case HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR: 2726 wc->status = IB_WC_REM_ABORT_ERR; 2727 break; 2728 default: 2729 wc->status = IB_WC_GENERAL_ERR; 2730 break; 2731 } 2732 2733 /* flush cqe if wc status is error, excluding flush error */ 2734 if ((wc->status != IB_WC_SUCCESS) && 2735 (wc->status != IB_WC_WR_FLUSH_ERR)) { 2736 attr_mask = IB_QP_STATE; 2737 attr.qp_state = IB_QPS_ERR; 2738 return hns_roce_v2_modify_qp(&(*cur_qp)->ibqp, 2739 &attr, attr_mask, 2740 (*cur_qp)->state, IB_QPS_ERR); 2741 } 2742 2743 if (wc->status == IB_WC_WR_FLUSH_ERR) 2744 return 0; 2745 2746 if (is_send) { 2747 wc->wc_flags = 0; 2748 /* SQ corresponding to CQE */ 2749 switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M, 2750 V2_CQE_BYTE_4_OPCODE_S) & 0x1f) { 2751 case HNS_ROCE_SQ_OPCODE_SEND: 2752 wc->opcode = IB_WC_SEND; 2753 break; 2754 case HNS_ROCE_SQ_OPCODE_SEND_WITH_INV: 2755 wc->opcode = IB_WC_SEND; 2756 break; 2757 case HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM: 2758 wc->opcode = IB_WC_SEND; 2759 wc->wc_flags |= IB_WC_WITH_IMM; 2760 break; 2761 case HNS_ROCE_SQ_OPCODE_RDMA_READ: 2762 wc->opcode = IB_WC_RDMA_READ; 2763 wc->byte_len = le32_to_cpu(cqe->byte_cnt); 2764 break; 2765 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE: 2766 wc->opcode = IB_WC_RDMA_WRITE; 2767 break; 2768 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM: 2769 wc->opcode = IB_WC_RDMA_WRITE; 2770 wc->wc_flags |= IB_WC_WITH_IMM; 2771 break; 2772 case HNS_ROCE_SQ_OPCODE_LOCAL_INV: 2773 wc->opcode = IB_WC_LOCAL_INV; 2774 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 2775 break; 2776 case HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP: 2777 wc->opcode = IB_WC_COMP_SWAP; 2778 wc->byte_len = 8; 2779 break; 2780 case HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD: 2781 wc->opcode = IB_WC_FETCH_ADD; 2782 wc->byte_len = 8; 2783 break; 2784 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP: 2785 wc->opcode = IB_WC_MASKED_COMP_SWAP; 2786 wc->byte_len = 8; 2787 break; 2788 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD: 2789 wc->opcode = IB_WC_MASKED_FETCH_ADD; 2790 wc->byte_len = 8; 2791 break; 2792 case HNS_ROCE_SQ_OPCODE_FAST_REG_WR: 2793 wc->opcode = IB_WC_REG_MR; 2794 break; 2795 case HNS_ROCE_SQ_OPCODE_BIND_MW: 2796 wc->opcode = IB_WC_REG_MR; 2797 break; 2798 default: 2799 wc->status = IB_WC_GENERAL_ERR; 2800 break; 2801 } 2802 } else { 2803 /* RQ correspond to CQE */ 2804 wc->byte_len = le32_to_cpu(cqe->byte_cnt); 2805 2806 opcode = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M, 2807 V2_CQE_BYTE_4_OPCODE_S); 2808 switch (opcode & 0x1f) { 2809 case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM: 2810 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 2811 wc->wc_flags = IB_WC_WITH_IMM; 2812 wc->ex.imm_data = 2813 cpu_to_be32(le32_to_cpu(cqe->immtdata)); 2814 break; 2815 case HNS_ROCE_V2_OPCODE_SEND: 2816 wc->opcode = IB_WC_RECV; 2817 wc->wc_flags = 0; 2818 break; 2819 case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM: 2820 wc->opcode = IB_WC_RECV; 2821 wc->wc_flags = IB_WC_WITH_IMM; 2822 wc->ex.imm_data = 2823 cpu_to_be32(le32_to_cpu(cqe->immtdata)); 2824 break; 2825 case HNS_ROCE_V2_OPCODE_SEND_WITH_INV: 2826 wc->opcode = IB_WC_RECV; 2827 wc->wc_flags = IB_WC_WITH_INVALIDATE; 2828 wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey); 2829 break; 2830 default: 2831 wc->status = IB_WC_GENERAL_ERR; 2832 break; 2833 } 2834 2835 if ((wc->qp->qp_type == IB_QPT_RC || 2836 wc->qp->qp_type == IB_QPT_UC) && 2837 (opcode == HNS_ROCE_V2_OPCODE_SEND || 2838 opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM || 2839 opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) && 2840 (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_RQ_INLINE_S))) { 2841 ret = hns_roce_handle_recv_inl_wqe(cqe, cur_qp, wc); 2842 if (ret) 2843 return -EAGAIN; 2844 } 2845 2846 wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M, 2847 V2_CQE_BYTE_32_SL_S); 2848 wc->src_qp = (u8)roce_get_field(cqe->byte_32, 2849 V2_CQE_BYTE_32_RMT_QPN_M, 2850 V2_CQE_BYTE_32_RMT_QPN_S); 2851 wc->slid = 0; 2852 wc->wc_flags |= (roce_get_bit(cqe->byte_32, 2853 V2_CQE_BYTE_32_GRH_S) ? 2854 IB_WC_GRH : 0); 2855 wc->port_num = roce_get_field(cqe->byte_32, 2856 V2_CQE_BYTE_32_PORTN_M, V2_CQE_BYTE_32_PORTN_S); 2857 wc->pkey_index = 0; 2858 memcpy(wc->smac, cqe->smac, 4); 2859 wc->smac[4] = roce_get_field(cqe->byte_28, 2860 V2_CQE_BYTE_28_SMAC_4_M, 2861 V2_CQE_BYTE_28_SMAC_4_S); 2862 wc->smac[5] = roce_get_field(cqe->byte_28, 2863 V2_CQE_BYTE_28_SMAC_5_M, 2864 V2_CQE_BYTE_28_SMAC_5_S); 2865 if (roce_get_bit(cqe->byte_28, V2_CQE_BYTE_28_VID_VLD_S)) { 2866 wc->vlan_id = (u16)roce_get_field(cqe->byte_28, 2867 V2_CQE_BYTE_28_VID_M, 2868 V2_CQE_BYTE_28_VID_S); 2869 } else { 2870 wc->vlan_id = 0xffff; 2871 } 2872 2873 wc->wc_flags |= (IB_WC_WITH_VLAN | IB_WC_WITH_SMAC); 2874 wc->network_hdr_type = roce_get_field(cqe->byte_28, 2875 V2_CQE_BYTE_28_PORT_TYPE_M, 2876 V2_CQE_BYTE_28_PORT_TYPE_S); 2877 } 2878 2879 return 0; 2880 } 2881 2882 static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries, 2883 struct ib_wc *wc) 2884 { 2885 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); 2886 struct hns_roce_qp *cur_qp = NULL; 2887 unsigned long flags; 2888 int npolled; 2889 2890 spin_lock_irqsave(&hr_cq->lock, flags); 2891 2892 for (npolled = 0; npolled < num_entries; ++npolled) { 2893 if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled)) 2894 break; 2895 } 2896 2897 if (npolled) { 2898 /* Memory barrier */ 2899 wmb(); 2900 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index); 2901 } 2902 2903 spin_unlock_irqrestore(&hr_cq->lock, flags); 2904 2905 return npolled; 2906 } 2907 2908 static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev, 2909 struct hns_roce_hem_table *table, int obj, 2910 int step_idx) 2911 { 2912 struct device *dev = hr_dev->dev; 2913 struct hns_roce_cmd_mailbox *mailbox; 2914 struct hns_roce_hem_iter iter; 2915 struct hns_roce_hem_mhop mhop; 2916 struct hns_roce_hem *hem; 2917 unsigned long mhop_obj = obj; 2918 int i, j, k; 2919 int ret = 0; 2920 u64 hem_idx = 0; 2921 u64 l1_idx = 0; 2922 u64 bt_ba = 0; 2923 u32 chunk_ba_num; 2924 u32 hop_num; 2925 u16 op = 0xff; 2926 2927 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) 2928 return 0; 2929 2930 hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); 2931 i = mhop.l0_idx; 2932 j = mhop.l1_idx; 2933 k = mhop.l2_idx; 2934 hop_num = mhop.hop_num; 2935 chunk_ba_num = mhop.bt_chunk_size / 8; 2936 2937 if (hop_num == 2) { 2938 hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num + 2939 k; 2940 l1_idx = i * chunk_ba_num + j; 2941 } else if (hop_num == 1) { 2942 hem_idx = i * chunk_ba_num + j; 2943 } else if (hop_num == HNS_ROCE_HOP_NUM_0) { 2944 hem_idx = i; 2945 } 2946 2947 switch (table->type) { 2948 case HEM_TYPE_QPC: 2949 op = HNS_ROCE_CMD_WRITE_QPC_BT0; 2950 break; 2951 case HEM_TYPE_MTPT: 2952 op = HNS_ROCE_CMD_WRITE_MPT_BT0; 2953 break; 2954 case HEM_TYPE_CQC: 2955 op = HNS_ROCE_CMD_WRITE_CQC_BT0; 2956 break; 2957 case HEM_TYPE_SRQC: 2958 op = HNS_ROCE_CMD_WRITE_SRQC_BT0; 2959 break; 2960 case HEM_TYPE_SCCC: 2961 op = HNS_ROCE_CMD_WRITE_SCCC_BT0; 2962 break; 2963 case HEM_TYPE_QPC_TIMER: 2964 op = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0; 2965 break; 2966 case HEM_TYPE_CQC_TIMER: 2967 op = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0; 2968 break; 2969 default: 2970 dev_warn(dev, "Table %d not to be written by mailbox!\n", 2971 table->type); 2972 return 0; 2973 } 2974 2975 if (table->type == HEM_TYPE_SCCC && step_idx) 2976 return 0; 2977 2978 op += step_idx; 2979 2980 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 2981 if (IS_ERR(mailbox)) 2982 return PTR_ERR(mailbox); 2983 2984 if (table->type == HEM_TYPE_SCCC) 2985 obj = mhop.l0_idx; 2986 2987 if (check_whether_last_step(hop_num, step_idx)) { 2988 hem = table->hem[hem_idx]; 2989 for (hns_roce_hem_first(hem, &iter); 2990 !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) { 2991 bt_ba = hns_roce_hem_addr(&iter); 2992 2993 /* configure the ba, tag, and op */ 2994 ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, 2995 obj, 0, op, 2996 HNS_ROCE_CMD_TIMEOUT_MSECS); 2997 } 2998 } else { 2999 if (step_idx == 0) 3000 bt_ba = table->bt_l0_dma_addr[i]; 3001 else if (step_idx == 1 && hop_num == 2) 3002 bt_ba = table->bt_l1_dma_addr[l1_idx]; 3003 3004 /* configure the ba, tag, and op */ 3005 ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj, 3006 0, op, HNS_ROCE_CMD_TIMEOUT_MSECS); 3007 } 3008 3009 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 3010 return ret; 3011 } 3012 3013 static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev, 3014 struct hns_roce_hem_table *table, int obj, 3015 int step_idx) 3016 { 3017 struct device *dev = hr_dev->dev; 3018 struct hns_roce_cmd_mailbox *mailbox; 3019 int ret; 3020 u16 op = 0xff; 3021 3022 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) 3023 return 0; 3024 3025 switch (table->type) { 3026 case HEM_TYPE_QPC: 3027 op = HNS_ROCE_CMD_DESTROY_QPC_BT0; 3028 break; 3029 case HEM_TYPE_MTPT: 3030 op = HNS_ROCE_CMD_DESTROY_MPT_BT0; 3031 break; 3032 case HEM_TYPE_CQC: 3033 op = HNS_ROCE_CMD_DESTROY_CQC_BT0; 3034 break; 3035 case HEM_TYPE_SCCC: 3036 case HEM_TYPE_QPC_TIMER: 3037 case HEM_TYPE_CQC_TIMER: 3038 break; 3039 case HEM_TYPE_SRQC: 3040 op = HNS_ROCE_CMD_DESTROY_SRQC_BT0; 3041 break; 3042 default: 3043 dev_warn(dev, "Table %d not to be destroyed by mailbox!\n", 3044 table->type); 3045 return 0; 3046 } 3047 3048 if (table->type == HEM_TYPE_SCCC || 3049 table->type == HEM_TYPE_QPC_TIMER || 3050 table->type == HEM_TYPE_CQC_TIMER) 3051 return 0; 3052 3053 op += step_idx; 3054 3055 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 3056 if (IS_ERR(mailbox)) 3057 return PTR_ERR(mailbox); 3058 3059 /* configure the tag and op */ 3060 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, obj, 0, op, 3061 HNS_ROCE_CMD_TIMEOUT_MSECS); 3062 3063 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 3064 return ret; 3065 } 3066 3067 static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev, 3068 enum ib_qp_state cur_state, 3069 enum ib_qp_state new_state, 3070 struct hns_roce_v2_qp_context *context, 3071 struct hns_roce_qp *hr_qp) 3072 { 3073 struct hns_roce_cmd_mailbox *mailbox; 3074 int ret; 3075 3076 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 3077 if (IS_ERR(mailbox)) 3078 return PTR_ERR(mailbox); 3079 3080 memcpy(mailbox->buf, context, sizeof(*context) * 2); 3081 3082 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0, 3083 HNS_ROCE_CMD_MODIFY_QPC, 3084 HNS_ROCE_CMD_TIMEOUT_MSECS); 3085 3086 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 3087 3088 return ret; 3089 } 3090 3091 static void set_access_flags(struct hns_roce_qp *hr_qp, 3092 struct hns_roce_v2_qp_context *context, 3093 struct hns_roce_v2_qp_context *qpc_mask, 3094 const struct ib_qp_attr *attr, int attr_mask) 3095 { 3096 u8 dest_rd_atomic; 3097 u32 access_flags; 3098 3099 dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ? 3100 attr->max_dest_rd_atomic : hr_qp->resp_depth; 3101 3102 access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ? 3103 attr->qp_access_flags : hr_qp->atomic_rd_en; 3104 3105 if (!dest_rd_atomic) 3106 access_flags &= IB_ACCESS_REMOTE_WRITE; 3107 3108 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 3109 !!(access_flags & IB_ACCESS_REMOTE_READ)); 3110 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 0); 3111 3112 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 3113 !!(access_flags & IB_ACCESS_REMOTE_WRITE)); 3114 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 0); 3115 3116 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 3117 !!(access_flags & IB_ACCESS_REMOTE_ATOMIC)); 3118 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0); 3119 } 3120 3121 static void modify_qp_reset_to_init(struct ib_qp *ibqp, 3122 const struct ib_qp_attr *attr, 3123 int attr_mask, 3124 struct hns_roce_v2_qp_context *context, 3125 struct hns_roce_v2_qp_context *qpc_mask) 3126 { 3127 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 3128 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 3129 3130 /* 3131 * In v2 engine, software pass context and context mask to hardware 3132 * when modifying qp. If software need modify some fields in context, 3133 * we should set all bits of the relevant fields in context mask to 3134 * 0 at the same time, else set them to 0x1. 3135 */ 3136 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M, 3137 V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type)); 3138 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M, 3139 V2_QPC_BYTE_4_TST_S, 0); 3140 3141 if (ibqp->qp_type == IB_QPT_GSI) 3142 roce_set_field(context->byte_4_sqpn_tst, 3143 V2_QPC_BYTE_4_SGE_SHIFT_M, 3144 V2_QPC_BYTE_4_SGE_SHIFT_S, 3145 ilog2((unsigned int)hr_qp->sge.sge_cnt)); 3146 else 3147 roce_set_field(context->byte_4_sqpn_tst, 3148 V2_QPC_BYTE_4_SGE_SHIFT_M, 3149 V2_QPC_BYTE_4_SGE_SHIFT_S, 3150 hr_qp->sq.max_gs > 2 ? 3151 ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0); 3152 3153 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M, 3154 V2_QPC_BYTE_4_SGE_SHIFT_S, 0); 3155 3156 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M, 3157 V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn); 3158 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M, 3159 V2_QPC_BYTE_4_SQPN_S, 0); 3160 3161 roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M, 3162 V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn); 3163 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M, 3164 V2_QPC_BYTE_16_PD_S, 0); 3165 3166 roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M, 3167 V2_QPC_BYTE_20_RQWS_S, ilog2(hr_qp->rq.max_gs)); 3168 roce_set_field(qpc_mask->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M, 3169 V2_QPC_BYTE_20_RQWS_S, 0); 3170 3171 roce_set_field(context->byte_20_smac_sgid_idx, 3172 V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 3173 ilog2((unsigned int)hr_qp->sq.wqe_cnt)); 3174 roce_set_field(qpc_mask->byte_20_smac_sgid_idx, 3175 V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0); 3176 3177 roce_set_field(context->byte_20_smac_sgid_idx, 3178 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 3179 (hr_qp->ibqp.qp_type == IB_QPT_XRC_INI || 3180 hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT || ibqp->srq) ? 0 : 3181 ilog2((unsigned int)hr_qp->rq.wqe_cnt)); 3182 roce_set_field(qpc_mask->byte_20_smac_sgid_idx, 3183 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0); 3184 3185 /* No VLAN need to set 0xFFF */ 3186 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M, 3187 V2_QPC_BYTE_24_VLAN_ID_S, 0xfff); 3188 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M, 3189 V2_QPC_BYTE_24_VLAN_ID_S, 0); 3190 3191 /* 3192 * Set some fields in context to zero, Because the default values 3193 * of all fields in context are zero, we need not set them to 0 again. 3194 * but we should set the relevant fields of context mask to 0. 3195 */ 3196 roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_TX_ERR_S, 0); 3197 roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_RX_ERR_S, 0); 3198 roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_TX_ERR_S, 0); 3199 roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_RX_ERR_S, 0); 3200 3201 roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_TEMPID_M, 3202 V2_QPC_BYTE_60_TEMPID_S, 0); 3203 3204 roce_set_field(qpc_mask->byte_60_qpst_tempid, 3205 V2_QPC_BYTE_60_SCC_TOKEN_M, V2_QPC_BYTE_60_SCC_TOKEN_S, 3206 0); 3207 roce_set_bit(qpc_mask->byte_60_qpst_tempid, 3208 V2_QPC_BYTE_60_SQ_DB_DOING_S, 0); 3209 roce_set_bit(qpc_mask->byte_60_qpst_tempid, 3210 V2_QPC_BYTE_60_RQ_DB_DOING_S, 0); 3211 roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CNP_TX_FLAG_S, 0); 3212 roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CE_FLAG_S, 0); 3213 3214 if (hr_qp->rdb_en) { 3215 roce_set_bit(context->byte_68_rq_db, 3216 V2_QPC_BYTE_68_RQ_RECORD_EN_S, 1); 3217 roce_set_bit(qpc_mask->byte_68_rq_db, 3218 V2_QPC_BYTE_68_RQ_RECORD_EN_S, 0); 3219 } 3220 3221 roce_set_field(context->byte_68_rq_db, 3222 V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M, 3223 V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S, 3224 ((u32)hr_qp->rdb.dma) >> 1); 3225 roce_set_field(qpc_mask->byte_68_rq_db, 3226 V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M, 3227 V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S, 0); 3228 context->rq_db_record_addr = hr_qp->rdb.dma >> 32; 3229 qpc_mask->rq_db_record_addr = 0; 3230 3231 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 3232 (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) ? 1 : 0); 3233 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 0); 3234 3235 roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M, 3236 V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn); 3237 roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M, 3238 V2_QPC_BYTE_80_RX_CQN_S, 0); 3239 if (ibqp->srq) { 3240 roce_set_field(context->byte_76_srqn_op_en, 3241 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 3242 to_hr_srq(ibqp->srq)->srqn); 3243 roce_set_field(qpc_mask->byte_76_srqn_op_en, 3244 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0); 3245 roce_set_bit(context->byte_76_srqn_op_en, 3246 V2_QPC_BYTE_76_SRQ_EN_S, 1); 3247 roce_set_bit(qpc_mask->byte_76_srqn_op_en, 3248 V2_QPC_BYTE_76_SRQ_EN_S, 0); 3249 } 3250 3251 roce_set_field(qpc_mask->byte_84_rq_ci_pi, 3252 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, 3253 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0); 3254 roce_set_field(qpc_mask->byte_84_rq_ci_pi, 3255 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M, 3256 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0); 3257 3258 roce_set_field(qpc_mask->byte_92_srq_info, V2_QPC_BYTE_92_SRQ_INFO_M, 3259 V2_QPC_BYTE_92_SRQ_INFO_S, 0); 3260 3261 roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M, 3262 V2_QPC_BYTE_96_RX_REQ_MSN_S, 0); 3263 3264 roce_set_field(qpc_mask->byte_104_rq_sge, 3265 V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_M, 3266 V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_S, 0); 3267 3268 roce_set_bit(qpc_mask->byte_108_rx_reqepsn, 3269 V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0); 3270 roce_set_field(qpc_mask->byte_108_rx_reqepsn, 3271 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M, 3272 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0); 3273 roce_set_bit(qpc_mask->byte_108_rx_reqepsn, 3274 V2_QPC_BYTE_108_RX_REQ_RNR_S, 0); 3275 3276 qpc_mask->rq_rnr_timer = 0; 3277 qpc_mask->rx_msg_len = 0; 3278 qpc_mask->rx_rkey_pkt_info = 0; 3279 qpc_mask->rx_va = 0; 3280 3281 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M, 3282 V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0); 3283 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M, 3284 V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0); 3285 3286 roce_set_bit(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RQ_RTY_WAIT_DO_S, 3287 0); 3288 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_HEAD_M, 3289 V2_QPC_BYTE_140_RAQ_TRRL_HEAD_S, 0); 3290 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_TAIL_M, 3291 V2_QPC_BYTE_140_RAQ_TRRL_TAIL_S, 0); 3292 3293 roce_set_field(qpc_mask->byte_144_raq, 3294 V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_M, 3295 V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_S, 0); 3296 roce_set_field(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RAQ_CREDIT_M, 3297 V2_QPC_BYTE_144_RAQ_CREDIT_S, 0); 3298 roce_set_bit(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RESP_RTY_FLG_S, 0); 3299 3300 roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RQ_MSN_M, 3301 V2_QPC_BYTE_148_RQ_MSN_S, 0); 3302 roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RAQ_SYNDROME_M, 3303 V2_QPC_BYTE_148_RAQ_SYNDROME_S, 0); 3304 3305 roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M, 3306 V2_QPC_BYTE_152_RAQ_PSN_S, 0); 3307 roce_set_field(qpc_mask->byte_152_raq, 3308 V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_M, 3309 V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_S, 0); 3310 3311 roce_set_field(qpc_mask->byte_156_raq, V2_QPC_BYTE_156_RAQ_USE_PKTN_M, 3312 V2_QPC_BYTE_156_RAQ_USE_PKTN_S, 0); 3313 3314 roce_set_field(qpc_mask->byte_160_sq_ci_pi, 3315 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M, 3316 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0); 3317 roce_set_field(qpc_mask->byte_160_sq_ci_pi, 3318 V2_QPC_BYTE_160_SQ_CONSUMER_IDX_M, 3319 V2_QPC_BYTE_160_SQ_CONSUMER_IDX_S, 0); 3320 3321 roce_set_bit(qpc_mask->byte_168_irrl_idx, 3322 V2_QPC_BYTE_168_POLL_DB_WAIT_DO_S, 0); 3323 roce_set_bit(qpc_mask->byte_168_irrl_idx, 3324 V2_QPC_BYTE_168_SCC_TOKEN_FORBID_SQ_DEQ_S, 0); 3325 roce_set_bit(qpc_mask->byte_168_irrl_idx, 3326 V2_QPC_BYTE_168_WAIT_ACK_TIMEOUT_S, 0); 3327 roce_set_bit(qpc_mask->byte_168_irrl_idx, 3328 V2_QPC_BYTE_168_MSG_RTY_LP_FLG_S, 0); 3329 roce_set_bit(qpc_mask->byte_168_irrl_idx, 3330 V2_QPC_BYTE_168_SQ_INVLD_FLG_S, 0); 3331 roce_set_field(qpc_mask->byte_168_irrl_idx, 3332 V2_QPC_BYTE_168_IRRL_IDX_LSB_M, 3333 V2_QPC_BYTE_168_IRRL_IDX_LSB_S, 0); 3334 3335 roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M, 3336 V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 4); 3337 roce_set_field(qpc_mask->byte_172_sq_psn, 3338 V2_QPC_BYTE_172_ACK_REQ_FREQ_M, 3339 V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 0); 3340 3341 roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_MSG_RNR_FLG_S, 3342 0); 3343 3344 roce_set_bit(context->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 1); 3345 roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 0); 3346 3347 roce_set_field(qpc_mask->byte_176_msg_pktn, 3348 V2_QPC_BYTE_176_MSG_USE_PKTN_M, 3349 V2_QPC_BYTE_176_MSG_USE_PKTN_S, 0); 3350 roce_set_field(qpc_mask->byte_176_msg_pktn, 3351 V2_QPC_BYTE_176_IRRL_HEAD_PRE_M, 3352 V2_QPC_BYTE_176_IRRL_HEAD_PRE_S, 0); 3353 3354 roce_set_field(qpc_mask->byte_184_irrl_idx, 3355 V2_QPC_BYTE_184_IRRL_IDX_MSB_M, 3356 V2_QPC_BYTE_184_IRRL_IDX_MSB_S, 0); 3357 3358 qpc_mask->cur_sge_offset = 0; 3359 3360 roce_set_field(qpc_mask->byte_192_ext_sge, 3361 V2_QPC_BYTE_192_CUR_SGE_IDX_M, 3362 V2_QPC_BYTE_192_CUR_SGE_IDX_S, 0); 3363 roce_set_field(qpc_mask->byte_192_ext_sge, 3364 V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_M, 3365 V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_S, 0); 3366 3367 roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M, 3368 V2_QPC_BYTE_196_IRRL_HEAD_S, 0); 3369 3370 roce_set_field(qpc_mask->byte_200_sq_max, V2_QPC_BYTE_200_SQ_MAX_IDX_M, 3371 V2_QPC_BYTE_200_SQ_MAX_IDX_S, 0); 3372 roce_set_field(qpc_mask->byte_200_sq_max, 3373 V2_QPC_BYTE_200_LCL_OPERATED_CNT_M, 3374 V2_QPC_BYTE_200_LCL_OPERATED_CNT_S, 0); 3375 3376 roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RNR_FLG_S, 0); 3377 roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RTY_FLG_S, 0); 3378 3379 roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M, 3380 V2_QPC_BYTE_212_CHECK_FLG_S, 0); 3381 3382 qpc_mask->sq_timer = 0; 3383 3384 roce_set_field(qpc_mask->byte_220_retry_psn_msn, 3385 V2_QPC_BYTE_220_RETRY_MSG_MSN_M, 3386 V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0); 3387 roce_set_field(qpc_mask->byte_232_irrl_sge, 3388 V2_QPC_BYTE_232_IRRL_SGE_IDX_M, 3389 V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0); 3390 3391 roce_set_bit(qpc_mask->byte_232_irrl_sge, V2_QPC_BYTE_232_SO_LP_VLD_S, 3392 0); 3393 roce_set_bit(qpc_mask->byte_232_irrl_sge, 3394 V2_QPC_BYTE_232_FENCE_LP_VLD_S, 0); 3395 roce_set_bit(qpc_mask->byte_232_irrl_sge, V2_QPC_BYTE_232_IRRL_LP_VLD_S, 3396 0); 3397 3398 qpc_mask->irrl_cur_sge_offset = 0; 3399 3400 roce_set_field(qpc_mask->byte_240_irrl_tail, 3401 V2_QPC_BYTE_240_IRRL_TAIL_REAL_M, 3402 V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0); 3403 roce_set_field(qpc_mask->byte_240_irrl_tail, 3404 V2_QPC_BYTE_240_IRRL_TAIL_RD_M, 3405 V2_QPC_BYTE_240_IRRL_TAIL_RD_S, 0); 3406 roce_set_field(qpc_mask->byte_240_irrl_tail, 3407 V2_QPC_BYTE_240_RX_ACK_MSN_M, 3408 V2_QPC_BYTE_240_RX_ACK_MSN_S, 0); 3409 3410 roce_set_field(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_M, 3411 V2_QPC_BYTE_248_IRRL_PSN_S, 0); 3412 roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_ACK_PSN_ERR_S, 3413 0); 3414 roce_set_field(qpc_mask->byte_248_ack_psn, 3415 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M, 3416 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0); 3417 roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_VLD_S, 3418 0); 3419 roce_set_bit(qpc_mask->byte_248_ack_psn, 3420 V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0); 3421 roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_CQ_ERR_IND_S, 3422 0); 3423 3424 hr_qp->access_flags = attr->qp_access_flags; 3425 roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M, 3426 V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn); 3427 roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M, 3428 V2_QPC_BYTE_252_TX_CQN_S, 0); 3429 3430 roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_ERR_TYPE_M, 3431 V2_QPC_BYTE_252_ERR_TYPE_S, 0); 3432 3433 roce_set_field(qpc_mask->byte_256_sqflush_rqcqe, 3434 V2_QPC_BYTE_256_RQ_CQE_IDX_M, 3435 V2_QPC_BYTE_256_RQ_CQE_IDX_S, 0); 3436 roce_set_field(qpc_mask->byte_256_sqflush_rqcqe, 3437 V2_QPC_BYTE_256_SQ_FLUSH_IDX_M, 3438 V2_QPC_BYTE_256_SQ_FLUSH_IDX_S, 0); 3439 } 3440 3441 static void modify_qp_init_to_init(struct ib_qp *ibqp, 3442 const struct ib_qp_attr *attr, int attr_mask, 3443 struct hns_roce_v2_qp_context *context, 3444 struct hns_roce_v2_qp_context *qpc_mask) 3445 { 3446 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 3447 3448 /* 3449 * In v2 engine, software pass context and context mask to hardware 3450 * when modifying qp. If software need modify some fields in context, 3451 * we should set all bits of the relevant fields in context mask to 3452 * 0 at the same time, else set them to 0x1. 3453 */ 3454 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M, 3455 V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type)); 3456 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M, 3457 V2_QPC_BYTE_4_TST_S, 0); 3458 3459 if (ibqp->qp_type == IB_QPT_GSI) 3460 roce_set_field(context->byte_4_sqpn_tst, 3461 V2_QPC_BYTE_4_SGE_SHIFT_M, 3462 V2_QPC_BYTE_4_SGE_SHIFT_S, 3463 ilog2((unsigned int)hr_qp->sge.sge_cnt)); 3464 else 3465 roce_set_field(context->byte_4_sqpn_tst, 3466 V2_QPC_BYTE_4_SGE_SHIFT_M, 3467 V2_QPC_BYTE_4_SGE_SHIFT_S, 3468 hr_qp->sq.max_gs > 3469 HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE ? 3470 ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0); 3471 3472 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M, 3473 V2_QPC_BYTE_4_SGE_SHIFT_S, 0); 3474 3475 if (attr_mask & IB_QP_ACCESS_FLAGS) { 3476 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 3477 !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ)); 3478 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 3479 0); 3480 3481 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 3482 !!(attr->qp_access_flags & 3483 IB_ACCESS_REMOTE_WRITE)); 3484 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 3485 0); 3486 3487 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 3488 !!(attr->qp_access_flags & 3489 IB_ACCESS_REMOTE_ATOMIC)); 3490 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 3491 0); 3492 } else { 3493 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 3494 !!(hr_qp->access_flags & IB_ACCESS_REMOTE_READ)); 3495 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 3496 0); 3497 3498 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 3499 !!(hr_qp->access_flags & IB_ACCESS_REMOTE_WRITE)); 3500 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 3501 0); 3502 3503 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 3504 !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC)); 3505 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 3506 0); 3507 } 3508 3509 roce_set_field(context->byte_20_smac_sgid_idx, 3510 V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 3511 ilog2((unsigned int)hr_qp->sq.wqe_cnt)); 3512 roce_set_field(qpc_mask->byte_20_smac_sgid_idx, 3513 V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0); 3514 3515 roce_set_field(context->byte_20_smac_sgid_idx, 3516 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 3517 (hr_qp->ibqp.qp_type == IB_QPT_XRC_INI || 3518 hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT || ibqp->srq) ? 0 : 3519 ilog2((unsigned int)hr_qp->rq.wqe_cnt)); 3520 roce_set_field(qpc_mask->byte_20_smac_sgid_idx, 3521 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0); 3522 3523 roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M, 3524 V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn); 3525 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M, 3526 V2_QPC_BYTE_16_PD_S, 0); 3527 3528 roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M, 3529 V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn); 3530 roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M, 3531 V2_QPC_BYTE_80_RX_CQN_S, 0); 3532 3533 roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M, 3534 V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn); 3535 roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M, 3536 V2_QPC_BYTE_252_TX_CQN_S, 0); 3537 3538 if (ibqp->srq) { 3539 roce_set_bit(context->byte_76_srqn_op_en, 3540 V2_QPC_BYTE_76_SRQ_EN_S, 1); 3541 roce_set_bit(qpc_mask->byte_76_srqn_op_en, 3542 V2_QPC_BYTE_76_SRQ_EN_S, 0); 3543 roce_set_field(context->byte_76_srqn_op_en, 3544 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 3545 to_hr_srq(ibqp->srq)->srqn); 3546 roce_set_field(qpc_mask->byte_76_srqn_op_en, 3547 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0); 3548 } 3549 3550 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M, 3551 V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn); 3552 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M, 3553 V2_QPC_BYTE_4_SQPN_S, 0); 3554 3555 if (attr_mask & IB_QP_DEST_QPN) { 3556 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M, 3557 V2_QPC_BYTE_56_DQPN_S, hr_qp->qpn); 3558 roce_set_field(qpc_mask->byte_56_dqpn_err, 3559 V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0); 3560 } 3561 } 3562 3563 static bool check_wqe_rq_mtt_count(struct hns_roce_dev *hr_dev, 3564 struct hns_roce_qp *hr_qp, int mtt_cnt, 3565 u32 page_size) 3566 { 3567 struct device *dev = hr_dev->dev; 3568 3569 if (hr_qp->rq.wqe_cnt < 1) 3570 return true; 3571 3572 if (mtt_cnt < 1) { 3573 dev_err(dev, "qp(0x%lx) rqwqe buf ba find failed\n", 3574 hr_qp->qpn); 3575 return false; 3576 } 3577 3578 if (mtt_cnt < MTT_MIN_COUNT && 3579 (hr_qp->rq.offset + page_size) < hr_qp->buff_size) { 3580 dev_err(dev, "qp(0x%lx) next rqwqe buf ba find failed\n", 3581 hr_qp->qpn); 3582 return false; 3583 } 3584 3585 return true; 3586 } 3587 3588 static int modify_qp_init_to_rtr(struct ib_qp *ibqp, 3589 const struct ib_qp_attr *attr, int attr_mask, 3590 struct hns_roce_v2_qp_context *context, 3591 struct hns_roce_v2_qp_context *qpc_mask) 3592 { 3593 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 3594 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 3595 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 3596 struct device *dev = hr_dev->dev; 3597 u64 mtts[MTT_MIN_COUNT] = { 0 }; 3598 dma_addr_t dma_handle_3; 3599 dma_addr_t dma_handle_2; 3600 u64 wqe_sge_ba; 3601 u32 page_size; 3602 u8 port_num; 3603 u64 *mtts_3; 3604 u64 *mtts_2; 3605 int count; 3606 u8 *dmac; 3607 u8 *smac; 3608 int port; 3609 3610 /* Search qp buf's mtts */ 3611 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); 3612 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 3613 hr_qp->rq.offset / page_size, mtts, 3614 MTT_MIN_COUNT, &wqe_sge_ba); 3615 if (!ibqp->srq) 3616 if (!check_wqe_rq_mtt_count(hr_dev, hr_qp, count, page_size)) 3617 return -EINVAL; 3618 3619 /* Search IRRL's mtts */ 3620 mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table, 3621 hr_qp->qpn, &dma_handle_2); 3622 if (!mtts_2) { 3623 dev_err(dev, "qp irrl_table find failed\n"); 3624 return -EINVAL; 3625 } 3626 3627 /* Search TRRL's mtts */ 3628 mtts_3 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table, 3629 hr_qp->qpn, &dma_handle_3); 3630 if (!mtts_3) { 3631 dev_err(dev, "qp trrl_table find failed\n"); 3632 return -EINVAL; 3633 } 3634 3635 if (attr_mask & IB_QP_ALT_PATH) { 3636 dev_err(dev, "INIT2RTR attr_mask (0x%x) error\n", attr_mask); 3637 return -EINVAL; 3638 } 3639 3640 dmac = (u8 *)attr->ah_attr.roce.dmac; 3641 context->wqe_sge_ba = (u32)(wqe_sge_ba >> 3); 3642 qpc_mask->wqe_sge_ba = 0; 3643 3644 /* 3645 * In v2 engine, software pass context and context mask to hardware 3646 * when modifying qp. If software need modify some fields in context, 3647 * we should set all bits of the relevant fields in context mask to 3648 * 0 at the same time, else set them to 0x1. 3649 */ 3650 roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M, 3651 V2_QPC_BYTE_12_WQE_SGE_BA_S, wqe_sge_ba >> (32 + 3)); 3652 roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M, 3653 V2_QPC_BYTE_12_WQE_SGE_BA_S, 0); 3654 3655 roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M, 3656 V2_QPC_BYTE_12_SQ_HOP_NUM_S, 3657 hr_dev->caps.wqe_sq_hop_num == HNS_ROCE_HOP_NUM_0 ? 3658 0 : hr_dev->caps.wqe_sq_hop_num); 3659 roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M, 3660 V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0); 3661 3662 roce_set_field(context->byte_20_smac_sgid_idx, 3663 V2_QPC_BYTE_20_SGE_HOP_NUM_M, 3664 V2_QPC_BYTE_20_SGE_HOP_NUM_S, 3665 ((ibqp->qp_type == IB_QPT_GSI) || 3666 hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ? 3667 hr_dev->caps.wqe_sge_hop_num : 0); 3668 roce_set_field(qpc_mask->byte_20_smac_sgid_idx, 3669 V2_QPC_BYTE_20_SGE_HOP_NUM_M, 3670 V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0); 3671 3672 roce_set_field(context->byte_20_smac_sgid_idx, 3673 V2_QPC_BYTE_20_RQ_HOP_NUM_M, 3674 V2_QPC_BYTE_20_RQ_HOP_NUM_S, 3675 hr_dev->caps.wqe_rq_hop_num == HNS_ROCE_HOP_NUM_0 ? 3676 0 : hr_dev->caps.wqe_rq_hop_num); 3677 roce_set_field(qpc_mask->byte_20_smac_sgid_idx, 3678 V2_QPC_BYTE_20_RQ_HOP_NUM_M, 3679 V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0); 3680 3681 roce_set_field(context->byte_16_buf_ba_pg_sz, 3682 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M, 3683 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 3684 hr_qp->wqe_bt_pg_shift + PG_SHIFT_OFFSET); 3685 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, 3686 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M, 3687 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0); 3688 3689 roce_set_field(context->byte_16_buf_ba_pg_sz, 3690 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M, 3691 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 3692 hr_dev->caps.mtt_buf_pg_sz + PG_SHIFT_OFFSET); 3693 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, 3694 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M, 3695 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0); 3696 3697 context->rq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT); 3698 qpc_mask->rq_cur_blk_addr = 0; 3699 3700 roce_set_field(context->byte_92_srq_info, 3701 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M, 3702 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 3703 mtts[0] >> (32 + PAGE_ADDR_SHIFT)); 3704 roce_set_field(qpc_mask->byte_92_srq_info, 3705 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M, 3706 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0); 3707 3708 context->rq_nxt_blk_addr = (u32)(mtts[1] >> PAGE_ADDR_SHIFT); 3709 qpc_mask->rq_nxt_blk_addr = 0; 3710 3711 roce_set_field(context->byte_104_rq_sge, 3712 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M, 3713 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 3714 mtts[1] >> (32 + PAGE_ADDR_SHIFT)); 3715 roce_set_field(qpc_mask->byte_104_rq_sge, 3716 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M, 3717 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0); 3718 3719 roce_set_field(context->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M, 3720 V2_QPC_BYTE_132_TRRL_BA_S, dma_handle_3 >> 4); 3721 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M, 3722 V2_QPC_BYTE_132_TRRL_BA_S, 0); 3723 context->trrl_ba = (u32)(dma_handle_3 >> (16 + 4)); 3724 qpc_mask->trrl_ba = 0; 3725 roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M, 3726 V2_QPC_BYTE_140_TRRL_BA_S, 3727 (u32)(dma_handle_3 >> (32 + 16 + 4))); 3728 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M, 3729 V2_QPC_BYTE_140_TRRL_BA_S, 0); 3730 3731 context->irrl_ba = (u32)(dma_handle_2 >> 6); 3732 qpc_mask->irrl_ba = 0; 3733 roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M, 3734 V2_QPC_BYTE_208_IRRL_BA_S, 3735 dma_handle_2 >> (32 + 6)); 3736 roce_set_field(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M, 3737 V2_QPC_BYTE_208_IRRL_BA_S, 0); 3738 3739 roce_set_bit(context->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 1); 3740 roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 0); 3741 3742 roce_set_bit(context->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S, 3743 hr_qp->sq_signal_bits); 3744 roce_set_bit(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S, 3745 0); 3746 3747 port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port; 3748 3749 smac = (u8 *)hr_dev->dev_addr[port]; 3750 /* when dmac equals smac or loop_idc is 1, it should loopback */ 3751 if (ether_addr_equal_unaligned(dmac, smac) || 3752 hr_dev->loop_idc == 0x1) { 3753 roce_set_bit(context->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 1); 3754 roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 0); 3755 } 3756 3757 if (attr_mask & IB_QP_DEST_QPN) { 3758 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M, 3759 V2_QPC_BYTE_56_DQPN_S, attr->dest_qp_num); 3760 roce_set_field(qpc_mask->byte_56_dqpn_err, 3761 V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0); 3762 } 3763 3764 /* Configure GID index */ 3765 port_num = rdma_ah_get_port_num(&attr->ah_attr); 3766 roce_set_field(context->byte_20_smac_sgid_idx, 3767 V2_QPC_BYTE_20_SGID_IDX_M, 3768 V2_QPC_BYTE_20_SGID_IDX_S, 3769 hns_get_gid_index(hr_dev, port_num - 1, 3770 grh->sgid_index)); 3771 roce_set_field(qpc_mask->byte_20_smac_sgid_idx, 3772 V2_QPC_BYTE_20_SGID_IDX_M, 3773 V2_QPC_BYTE_20_SGID_IDX_S, 0); 3774 memcpy(&(context->dmac), dmac, sizeof(u32)); 3775 roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M, 3776 V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&dmac[4]))); 3777 qpc_mask->dmac = 0; 3778 roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M, 3779 V2_QPC_BYTE_52_DMAC_S, 0); 3780 3781 /* mtu*(2^LP_PKTN_INI) should not bigger than 1 message length 64kb */ 3782 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M, 3783 V2_QPC_BYTE_56_LP_PKTN_INI_S, 4); 3784 roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M, 3785 V2_QPC_BYTE_56_LP_PKTN_INI_S, 0); 3786 3787 if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD) 3788 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M, 3789 V2_QPC_BYTE_24_MTU_S, IB_MTU_4096); 3790 else if (attr_mask & IB_QP_PATH_MTU) 3791 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M, 3792 V2_QPC_BYTE_24_MTU_S, attr->path_mtu); 3793 3794 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M, 3795 V2_QPC_BYTE_24_MTU_S, 0); 3796 3797 roce_set_field(context->byte_84_rq_ci_pi, 3798 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, 3799 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head); 3800 roce_set_field(qpc_mask->byte_84_rq_ci_pi, 3801 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, 3802 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0); 3803 3804 roce_set_field(qpc_mask->byte_84_rq_ci_pi, 3805 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M, 3806 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0); 3807 roce_set_bit(qpc_mask->byte_108_rx_reqepsn, 3808 V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0); 3809 roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M, 3810 V2_QPC_BYTE_96_RX_REQ_MSN_S, 0); 3811 roce_set_field(qpc_mask->byte_108_rx_reqepsn, 3812 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M, 3813 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0); 3814 3815 context->rq_rnr_timer = 0; 3816 qpc_mask->rq_rnr_timer = 0; 3817 3818 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M, 3819 V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0); 3820 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M, 3821 V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0); 3822 3823 /* rocee send 2^lp_sgen_ini segs every time */ 3824 roce_set_field(context->byte_168_irrl_idx, 3825 V2_QPC_BYTE_168_LP_SGEN_INI_M, 3826 V2_QPC_BYTE_168_LP_SGEN_INI_S, 3); 3827 roce_set_field(qpc_mask->byte_168_irrl_idx, 3828 V2_QPC_BYTE_168_LP_SGEN_INI_M, 3829 V2_QPC_BYTE_168_LP_SGEN_INI_S, 0); 3830 3831 return 0; 3832 } 3833 3834 static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, 3835 const struct ib_qp_attr *attr, int attr_mask, 3836 struct hns_roce_v2_qp_context *context, 3837 struct hns_roce_v2_qp_context *qpc_mask) 3838 { 3839 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 3840 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 3841 struct device *dev = hr_dev->dev; 3842 u64 sge_cur_blk = 0; 3843 u64 sq_cur_blk = 0; 3844 u32 page_size; 3845 int count; 3846 3847 /* Search qp buf's mtts */ 3848 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL); 3849 if (count < 1) { 3850 dev_err(dev, "qp(0x%lx) buf pa find failed\n", hr_qp->qpn); 3851 return -EINVAL; 3852 } 3853 3854 if (hr_qp->sge.offset) { 3855 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); 3856 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 3857 hr_qp->sge.offset / page_size, 3858 &sge_cur_blk, 1, NULL); 3859 if (count < 1) { 3860 dev_err(dev, "qp(0x%lx) sge pa find failed\n", 3861 hr_qp->qpn); 3862 return -EINVAL; 3863 } 3864 } 3865 3866 /* Not support alternate path and path migration */ 3867 if ((attr_mask & IB_QP_ALT_PATH) || 3868 (attr_mask & IB_QP_PATH_MIG_STATE)) { 3869 dev_err(dev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask); 3870 return -EINVAL; 3871 } 3872 3873 /* 3874 * In v2 engine, software pass context and context mask to hardware 3875 * when modifying qp. If software need modify some fields in context, 3876 * we should set all bits of the relevant fields in context mask to 3877 * 0 at the same time, else set them to 0x1. 3878 */ 3879 context->sq_cur_blk_addr = (u32)(sq_cur_blk >> PAGE_ADDR_SHIFT); 3880 roce_set_field(context->byte_168_irrl_idx, 3881 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M, 3882 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 3883 sq_cur_blk >> (32 + PAGE_ADDR_SHIFT)); 3884 qpc_mask->sq_cur_blk_addr = 0; 3885 roce_set_field(qpc_mask->byte_168_irrl_idx, 3886 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M, 3887 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0); 3888 3889 context->sq_cur_sge_blk_addr = ((ibqp->qp_type == IB_QPT_GSI) || 3890 hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ? 3891 ((u32)(sge_cur_blk >> 3892 PAGE_ADDR_SHIFT)) : 0; 3893 roce_set_field(context->byte_184_irrl_idx, 3894 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M, 3895 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 3896 ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 3897 HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ? 3898 (sge_cur_blk >> 3899 (32 + PAGE_ADDR_SHIFT)) : 0); 3900 qpc_mask->sq_cur_sge_blk_addr = 0; 3901 roce_set_field(qpc_mask->byte_184_irrl_idx, 3902 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M, 3903 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0); 3904 3905 context->rx_sq_cur_blk_addr = (u32)(sq_cur_blk >> PAGE_ADDR_SHIFT); 3906 roce_set_field(context->byte_232_irrl_sge, 3907 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M, 3908 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 3909 sq_cur_blk >> (32 + PAGE_ADDR_SHIFT)); 3910 qpc_mask->rx_sq_cur_blk_addr = 0; 3911 roce_set_field(qpc_mask->byte_232_irrl_sge, 3912 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M, 3913 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 0); 3914 3915 /* 3916 * Set some fields in context to zero, Because the default values 3917 * of all fields in context are zero, we need not set them to 0 again. 3918 * but we should set the relevant fields of context mask to 0. 3919 */ 3920 roce_set_field(qpc_mask->byte_232_irrl_sge, 3921 V2_QPC_BYTE_232_IRRL_SGE_IDX_M, 3922 V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0); 3923 3924 roce_set_field(qpc_mask->byte_240_irrl_tail, 3925 V2_QPC_BYTE_240_RX_ACK_MSN_M, 3926 V2_QPC_BYTE_240_RX_ACK_MSN_S, 0); 3927 3928 roce_set_field(qpc_mask->byte_248_ack_psn, 3929 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M, 3930 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0); 3931 roce_set_bit(qpc_mask->byte_248_ack_psn, 3932 V2_QPC_BYTE_248_IRRL_PSN_VLD_S, 0); 3933 roce_set_field(qpc_mask->byte_248_ack_psn, 3934 V2_QPC_BYTE_248_IRRL_PSN_M, 3935 V2_QPC_BYTE_248_IRRL_PSN_S, 0); 3936 3937 roce_set_field(qpc_mask->byte_240_irrl_tail, 3938 V2_QPC_BYTE_240_IRRL_TAIL_REAL_M, 3939 V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0); 3940 3941 roce_set_field(qpc_mask->byte_220_retry_psn_msn, 3942 V2_QPC_BYTE_220_RETRY_MSG_MSN_M, 3943 V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0); 3944 3945 roce_set_bit(qpc_mask->byte_248_ack_psn, 3946 V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0); 3947 3948 roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M, 3949 V2_QPC_BYTE_212_CHECK_FLG_S, 0); 3950 3951 roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_LSN_M, 3952 V2_QPC_BYTE_212_LSN_S, 0x100); 3953 roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_LSN_M, 3954 V2_QPC_BYTE_212_LSN_S, 0); 3955 3956 roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M, 3957 V2_QPC_BYTE_196_IRRL_HEAD_S, 0); 3958 3959 return 0; 3960 } 3961 3962 static inline bool hns_roce_v2_check_qp_stat(enum ib_qp_state cur_state, 3963 enum ib_qp_state new_state) 3964 { 3965 3966 if ((cur_state != IB_QPS_RESET && 3967 (new_state == IB_QPS_ERR || new_state == IB_QPS_RESET)) || 3968 ((cur_state == IB_QPS_RTS || cur_state == IB_QPS_SQD) && 3969 (new_state == IB_QPS_RTS || new_state == IB_QPS_SQD)) || 3970 (cur_state == IB_QPS_SQE && new_state == IB_QPS_RTS)) 3971 return true; 3972 3973 return false; 3974 3975 } 3976 3977 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, 3978 const struct ib_qp_attr *attr, 3979 int attr_mask, enum ib_qp_state cur_state, 3980 enum ib_qp_state new_state) 3981 { 3982 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 3983 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 3984 struct hns_roce_v2_qp_context *context; 3985 struct hns_roce_v2_qp_context *qpc_mask; 3986 struct device *dev = hr_dev->dev; 3987 int ret = -EINVAL; 3988 3989 context = kcalloc(2, sizeof(*context), GFP_ATOMIC); 3990 if (!context) 3991 return -ENOMEM; 3992 3993 qpc_mask = context + 1; 3994 /* 3995 * In v2 engine, software pass context and context mask to hardware 3996 * when modifying qp. If software need modify some fields in context, 3997 * we should set all bits of the relevant fields in context mask to 3998 * 0 at the same time, else set them to 0x1. 3999 */ 4000 memset(qpc_mask, 0xff, sizeof(*qpc_mask)); 4001 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { 4002 memset(qpc_mask, 0, sizeof(*qpc_mask)); 4003 modify_qp_reset_to_init(ibqp, attr, attr_mask, context, 4004 qpc_mask); 4005 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) { 4006 modify_qp_init_to_init(ibqp, attr, attr_mask, context, 4007 qpc_mask); 4008 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) { 4009 ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context, 4010 qpc_mask); 4011 if (ret) 4012 goto out; 4013 } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) { 4014 ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context, 4015 qpc_mask); 4016 if (ret) 4017 goto out; 4018 } else if (hns_roce_v2_check_qp_stat(cur_state, new_state)) { 4019 /* Nothing */ 4020 ; 4021 } else { 4022 dev_err(dev, "Illegal state for QP!\n"); 4023 ret = -EINVAL; 4024 goto out; 4025 } 4026 4027 /* When QP state is err, SQ and RQ WQE should be flushed */ 4028 if (new_state == IB_QPS_ERR) { 4029 roce_set_field(context->byte_160_sq_ci_pi, 4030 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M, 4031 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 4032 hr_qp->sq.head); 4033 roce_set_field(qpc_mask->byte_160_sq_ci_pi, 4034 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M, 4035 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0); 4036 4037 if (!ibqp->srq) { 4038 roce_set_field(context->byte_84_rq_ci_pi, 4039 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, 4040 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 4041 hr_qp->rq.head); 4042 roce_set_field(qpc_mask->byte_84_rq_ci_pi, 4043 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, 4044 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0); 4045 } 4046 } 4047 4048 if (attr_mask & IB_QP_AV) { 4049 const struct ib_global_route *grh = 4050 rdma_ah_read_grh(&attr->ah_attr); 4051 const struct ib_gid_attr *gid_attr = NULL; 4052 int is_roce_protocol; 4053 u16 vlan = 0xffff; 4054 u8 ib_port; 4055 u8 hr_port; 4056 4057 ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num : 4058 hr_qp->port + 1; 4059 hr_port = ib_port - 1; 4060 is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) && 4061 rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH; 4062 4063 if (is_roce_protocol) { 4064 gid_attr = attr->ah_attr.grh.sgid_attr; 4065 ret = rdma_read_gid_l2_fields(gid_attr, &vlan, NULL); 4066 if (ret) 4067 goto out; 4068 } 4069 4070 if (vlan < VLAN_CFI_MASK) { 4071 roce_set_bit(context->byte_76_srqn_op_en, 4072 V2_QPC_BYTE_76_RQ_VLAN_EN_S, 1); 4073 roce_set_bit(qpc_mask->byte_76_srqn_op_en, 4074 V2_QPC_BYTE_76_RQ_VLAN_EN_S, 0); 4075 roce_set_bit(context->byte_168_irrl_idx, 4076 V2_QPC_BYTE_168_SQ_VLAN_EN_S, 1); 4077 roce_set_bit(qpc_mask->byte_168_irrl_idx, 4078 V2_QPC_BYTE_168_SQ_VLAN_EN_S, 0); 4079 } 4080 4081 roce_set_field(context->byte_24_mtu_tc, 4082 V2_QPC_BYTE_24_VLAN_ID_M, 4083 V2_QPC_BYTE_24_VLAN_ID_S, vlan); 4084 roce_set_field(qpc_mask->byte_24_mtu_tc, 4085 V2_QPC_BYTE_24_VLAN_ID_M, 4086 V2_QPC_BYTE_24_VLAN_ID_S, 0); 4087 4088 if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) { 4089 dev_err(hr_dev->dev, 4090 "sgid_index(%u) too large. max is %d\n", 4091 grh->sgid_index, 4092 hr_dev->caps.gid_table_len[hr_port]); 4093 ret = -EINVAL; 4094 goto out; 4095 } 4096 4097 if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) { 4098 dev_err(hr_dev->dev, "ah attr is not RDMA roce type\n"); 4099 ret = -EINVAL; 4100 goto out; 4101 } 4102 4103 roce_set_field(context->byte_52_udpspn_dmac, 4104 V2_QPC_BYTE_52_UDPSPN_M, V2_QPC_BYTE_52_UDPSPN_S, 4105 (gid_attr->gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP) ? 4106 0 : 0x12b7); 4107 4108 roce_set_field(qpc_mask->byte_52_udpspn_dmac, 4109 V2_QPC_BYTE_52_UDPSPN_M, 4110 V2_QPC_BYTE_52_UDPSPN_S, 0); 4111 4112 roce_set_field(context->byte_20_smac_sgid_idx, 4113 V2_QPC_BYTE_20_SGID_IDX_M, 4114 V2_QPC_BYTE_20_SGID_IDX_S, grh->sgid_index); 4115 4116 roce_set_field(qpc_mask->byte_20_smac_sgid_idx, 4117 V2_QPC_BYTE_20_SGID_IDX_M, 4118 V2_QPC_BYTE_20_SGID_IDX_S, 0); 4119 4120 roce_set_field(context->byte_24_mtu_tc, 4121 V2_QPC_BYTE_24_HOP_LIMIT_M, 4122 V2_QPC_BYTE_24_HOP_LIMIT_S, grh->hop_limit); 4123 roce_set_field(qpc_mask->byte_24_mtu_tc, 4124 V2_QPC_BYTE_24_HOP_LIMIT_M, 4125 V2_QPC_BYTE_24_HOP_LIMIT_S, 0); 4126 4127 if (hr_dev->pci_dev->revision == 0x21 && 4128 gid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) 4129 roce_set_field(context->byte_24_mtu_tc, 4130 V2_QPC_BYTE_24_TC_M, V2_QPC_BYTE_24_TC_S, 4131 grh->traffic_class >> 2); 4132 else 4133 roce_set_field(context->byte_24_mtu_tc, 4134 V2_QPC_BYTE_24_TC_M, V2_QPC_BYTE_24_TC_S, 4135 grh->traffic_class); 4136 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M, 4137 V2_QPC_BYTE_24_TC_S, 0); 4138 roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M, 4139 V2_QPC_BYTE_28_FL_S, grh->flow_label); 4140 roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_FL_M, 4141 V2_QPC_BYTE_28_FL_S, 0); 4142 memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw)); 4143 memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw)); 4144 roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M, 4145 V2_QPC_BYTE_28_SL_S, 4146 rdma_ah_get_sl(&attr->ah_attr)); 4147 roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M, 4148 V2_QPC_BYTE_28_SL_S, 0); 4149 hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); 4150 } 4151 4152 if (attr_mask & IB_QP_TIMEOUT) { 4153 if (attr->timeout < 31) { 4154 roce_set_field(context->byte_28_at_fl, 4155 V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S, 4156 attr->timeout); 4157 roce_set_field(qpc_mask->byte_28_at_fl, 4158 V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S, 4159 0); 4160 } else { 4161 dev_warn(dev, "Local ACK timeout shall be 0 to 30.\n"); 4162 } 4163 } 4164 4165 if (attr_mask & IB_QP_RETRY_CNT) { 4166 roce_set_field(context->byte_212_lsn, 4167 V2_QPC_BYTE_212_RETRY_NUM_INIT_M, 4168 V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 4169 attr->retry_cnt); 4170 roce_set_field(qpc_mask->byte_212_lsn, 4171 V2_QPC_BYTE_212_RETRY_NUM_INIT_M, 4172 V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 0); 4173 4174 roce_set_field(context->byte_212_lsn, 4175 V2_QPC_BYTE_212_RETRY_CNT_M, 4176 V2_QPC_BYTE_212_RETRY_CNT_S, 4177 attr->retry_cnt); 4178 roce_set_field(qpc_mask->byte_212_lsn, 4179 V2_QPC_BYTE_212_RETRY_CNT_M, 4180 V2_QPC_BYTE_212_RETRY_CNT_S, 0); 4181 } 4182 4183 if (attr_mask & IB_QP_RNR_RETRY) { 4184 roce_set_field(context->byte_244_rnr_rxack, 4185 V2_QPC_BYTE_244_RNR_NUM_INIT_M, 4186 V2_QPC_BYTE_244_RNR_NUM_INIT_S, attr->rnr_retry); 4187 roce_set_field(qpc_mask->byte_244_rnr_rxack, 4188 V2_QPC_BYTE_244_RNR_NUM_INIT_M, 4189 V2_QPC_BYTE_244_RNR_NUM_INIT_S, 0); 4190 4191 roce_set_field(context->byte_244_rnr_rxack, 4192 V2_QPC_BYTE_244_RNR_CNT_M, 4193 V2_QPC_BYTE_244_RNR_CNT_S, attr->rnr_retry); 4194 roce_set_field(qpc_mask->byte_244_rnr_rxack, 4195 V2_QPC_BYTE_244_RNR_CNT_M, 4196 V2_QPC_BYTE_244_RNR_CNT_S, 0); 4197 } 4198 4199 if (attr_mask & IB_QP_SQ_PSN) { 4200 roce_set_field(context->byte_172_sq_psn, 4201 V2_QPC_BYTE_172_SQ_CUR_PSN_M, 4202 V2_QPC_BYTE_172_SQ_CUR_PSN_S, attr->sq_psn); 4203 roce_set_field(qpc_mask->byte_172_sq_psn, 4204 V2_QPC_BYTE_172_SQ_CUR_PSN_M, 4205 V2_QPC_BYTE_172_SQ_CUR_PSN_S, 0); 4206 4207 roce_set_field(context->byte_196_sq_psn, 4208 V2_QPC_BYTE_196_SQ_MAX_PSN_M, 4209 V2_QPC_BYTE_196_SQ_MAX_PSN_S, attr->sq_psn); 4210 roce_set_field(qpc_mask->byte_196_sq_psn, 4211 V2_QPC_BYTE_196_SQ_MAX_PSN_M, 4212 V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0); 4213 4214 roce_set_field(context->byte_220_retry_psn_msn, 4215 V2_QPC_BYTE_220_RETRY_MSG_PSN_M, 4216 V2_QPC_BYTE_220_RETRY_MSG_PSN_S, attr->sq_psn); 4217 roce_set_field(qpc_mask->byte_220_retry_psn_msn, 4218 V2_QPC_BYTE_220_RETRY_MSG_PSN_M, 4219 V2_QPC_BYTE_220_RETRY_MSG_PSN_S, 0); 4220 4221 roce_set_field(context->byte_224_retry_msg, 4222 V2_QPC_BYTE_224_RETRY_MSG_PSN_M, 4223 V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 4224 attr->sq_psn >> V2_QPC_BYTE_220_RETRY_MSG_PSN_S); 4225 roce_set_field(qpc_mask->byte_224_retry_msg, 4226 V2_QPC_BYTE_224_RETRY_MSG_PSN_M, 4227 V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0); 4228 4229 roce_set_field(context->byte_224_retry_msg, 4230 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M, 4231 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 4232 attr->sq_psn); 4233 roce_set_field(qpc_mask->byte_224_retry_msg, 4234 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M, 4235 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 0); 4236 4237 roce_set_field(context->byte_244_rnr_rxack, 4238 V2_QPC_BYTE_244_RX_ACK_EPSN_M, 4239 V2_QPC_BYTE_244_RX_ACK_EPSN_S, attr->sq_psn); 4240 roce_set_field(qpc_mask->byte_244_rnr_rxack, 4241 V2_QPC_BYTE_244_RX_ACK_EPSN_M, 4242 V2_QPC_BYTE_244_RX_ACK_EPSN_S, 0); 4243 } 4244 4245 if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) && 4246 attr->max_dest_rd_atomic) { 4247 roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M, 4248 V2_QPC_BYTE_140_RR_MAX_S, 4249 fls(attr->max_dest_rd_atomic - 1)); 4250 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M, 4251 V2_QPC_BYTE_140_RR_MAX_S, 0); 4252 } 4253 4254 if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) { 4255 roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M, 4256 V2_QPC_BYTE_208_SR_MAX_S, 4257 fls(attr->max_rd_atomic - 1)); 4258 roce_set_field(qpc_mask->byte_208_irrl, 4259 V2_QPC_BYTE_208_SR_MAX_M, 4260 V2_QPC_BYTE_208_SR_MAX_S, 0); 4261 } 4262 4263 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) 4264 set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask); 4265 4266 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 4267 roce_set_field(context->byte_80_rnr_rx_cqn, 4268 V2_QPC_BYTE_80_MIN_RNR_TIME_M, 4269 V2_QPC_BYTE_80_MIN_RNR_TIME_S, 4270 attr->min_rnr_timer); 4271 roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, 4272 V2_QPC_BYTE_80_MIN_RNR_TIME_M, 4273 V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0); 4274 } 4275 4276 /* RC&UC required attr */ 4277 if (attr_mask & IB_QP_RQ_PSN) { 4278 roce_set_field(context->byte_108_rx_reqepsn, 4279 V2_QPC_BYTE_108_RX_REQ_EPSN_M, 4280 V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn); 4281 roce_set_field(qpc_mask->byte_108_rx_reqepsn, 4282 V2_QPC_BYTE_108_RX_REQ_EPSN_M, 4283 V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0); 4284 4285 roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M, 4286 V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1); 4287 roce_set_field(qpc_mask->byte_152_raq, 4288 V2_QPC_BYTE_152_RAQ_PSN_M, 4289 V2_QPC_BYTE_152_RAQ_PSN_S, 0); 4290 } 4291 4292 if (attr_mask & IB_QP_QKEY) { 4293 context->qkey_xrcd = attr->qkey; 4294 qpc_mask->qkey_xrcd = 0; 4295 hr_qp->qkey = attr->qkey; 4296 } 4297 4298 roce_set_bit(context->byte_108_rx_reqepsn, V2_QPC_BYTE_108_INV_CREDIT_S, 4299 ibqp->srq ? 1 : 0); 4300 roce_set_bit(qpc_mask->byte_108_rx_reqepsn, 4301 V2_QPC_BYTE_108_INV_CREDIT_S, 0); 4302 4303 /* Every status migrate must change state */ 4304 roce_set_field(context->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M, 4305 V2_QPC_BYTE_60_QP_ST_S, new_state); 4306 roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M, 4307 V2_QPC_BYTE_60_QP_ST_S, 0); 4308 4309 /* SW pass context to HW */ 4310 ret = hns_roce_v2_qp_modify(hr_dev, cur_state, new_state, 4311 context, hr_qp); 4312 if (ret) { 4313 dev_err(dev, "hns_roce_qp_modify failed(%d)\n", ret); 4314 goto out; 4315 } 4316 4317 hr_qp->state = new_state; 4318 4319 if (attr_mask & IB_QP_ACCESS_FLAGS) 4320 hr_qp->atomic_rd_en = attr->qp_access_flags; 4321 4322 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) 4323 hr_qp->resp_depth = attr->max_dest_rd_atomic; 4324 if (attr_mask & IB_QP_PORT) { 4325 hr_qp->port = attr->port_num - 1; 4326 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; 4327 } 4328 4329 if (new_state == IB_QPS_RESET && !ibqp->uobject) { 4330 hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn, 4331 ibqp->srq ? to_hr_srq(ibqp->srq) : NULL); 4332 if (ibqp->send_cq != ibqp->recv_cq) 4333 hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq), 4334 hr_qp->qpn, NULL); 4335 4336 hr_qp->rq.head = 0; 4337 hr_qp->rq.tail = 0; 4338 hr_qp->sq.head = 0; 4339 hr_qp->sq.tail = 0; 4340 hr_qp->sq_next_wqe = 0; 4341 hr_qp->next_sge = 0; 4342 if (hr_qp->rq.wqe_cnt) 4343 *hr_qp->rdb.db_record = 0; 4344 } 4345 4346 out: 4347 kfree(context); 4348 return ret; 4349 } 4350 4351 static inline enum ib_qp_state to_ib_qp_st(enum hns_roce_v2_qp_state state) 4352 { 4353 switch (state) { 4354 case HNS_ROCE_QP_ST_RST: return IB_QPS_RESET; 4355 case HNS_ROCE_QP_ST_INIT: return IB_QPS_INIT; 4356 case HNS_ROCE_QP_ST_RTR: return IB_QPS_RTR; 4357 case HNS_ROCE_QP_ST_RTS: return IB_QPS_RTS; 4358 case HNS_ROCE_QP_ST_SQ_DRAINING: 4359 case HNS_ROCE_QP_ST_SQD: return IB_QPS_SQD; 4360 case HNS_ROCE_QP_ST_SQER: return IB_QPS_SQE; 4361 case HNS_ROCE_QP_ST_ERR: return IB_QPS_ERR; 4362 default: return -1; 4363 } 4364 } 4365 4366 static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, 4367 struct hns_roce_qp *hr_qp, 4368 struct hns_roce_v2_qp_context *hr_context) 4369 { 4370 struct hns_roce_cmd_mailbox *mailbox; 4371 int ret; 4372 4373 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 4374 if (IS_ERR(mailbox)) 4375 return PTR_ERR(mailbox); 4376 4377 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0, 4378 HNS_ROCE_CMD_QUERY_QPC, 4379 HNS_ROCE_CMD_TIMEOUT_MSECS); 4380 if (ret) { 4381 dev_err(hr_dev->dev, "QUERY QP cmd process error\n"); 4382 goto out; 4383 } 4384 4385 memcpy(hr_context, mailbox->buf, sizeof(*hr_context)); 4386 4387 out: 4388 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 4389 return ret; 4390 } 4391 4392 static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 4393 int qp_attr_mask, 4394 struct ib_qp_init_attr *qp_init_attr) 4395 { 4396 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4397 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4398 struct hns_roce_v2_qp_context *context; 4399 struct device *dev = hr_dev->dev; 4400 int tmp_qp_state; 4401 int state; 4402 int ret; 4403 4404 context = kzalloc(sizeof(*context), GFP_KERNEL); 4405 if (!context) 4406 return -ENOMEM; 4407 4408 memset(qp_attr, 0, sizeof(*qp_attr)); 4409 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 4410 4411 mutex_lock(&hr_qp->mutex); 4412 4413 if (hr_qp->state == IB_QPS_RESET) { 4414 qp_attr->qp_state = IB_QPS_RESET; 4415 ret = 0; 4416 goto done; 4417 } 4418 4419 ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, context); 4420 if (ret) { 4421 dev_err(dev, "query qpc error\n"); 4422 ret = -EINVAL; 4423 goto out; 4424 } 4425 4426 state = roce_get_field(context->byte_60_qpst_tempid, 4427 V2_QPC_BYTE_60_QP_ST_M, V2_QPC_BYTE_60_QP_ST_S); 4428 tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state); 4429 if (tmp_qp_state == -1) { 4430 dev_err(dev, "Illegal ib_qp_state\n"); 4431 ret = -EINVAL; 4432 goto out; 4433 } 4434 hr_qp->state = (u8)tmp_qp_state; 4435 qp_attr->qp_state = (enum ib_qp_state)hr_qp->state; 4436 qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context->byte_24_mtu_tc, 4437 V2_QPC_BYTE_24_MTU_M, 4438 V2_QPC_BYTE_24_MTU_S); 4439 qp_attr->path_mig_state = IB_MIG_ARMED; 4440 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 4441 if (hr_qp->ibqp.qp_type == IB_QPT_UD) 4442 qp_attr->qkey = V2_QKEY_VAL; 4443 4444 qp_attr->rq_psn = roce_get_field(context->byte_108_rx_reqepsn, 4445 V2_QPC_BYTE_108_RX_REQ_EPSN_M, 4446 V2_QPC_BYTE_108_RX_REQ_EPSN_S); 4447 qp_attr->sq_psn = (u32)roce_get_field(context->byte_172_sq_psn, 4448 V2_QPC_BYTE_172_SQ_CUR_PSN_M, 4449 V2_QPC_BYTE_172_SQ_CUR_PSN_S); 4450 qp_attr->dest_qp_num = (u8)roce_get_field(context->byte_56_dqpn_err, 4451 V2_QPC_BYTE_56_DQPN_M, 4452 V2_QPC_BYTE_56_DQPN_S); 4453 qp_attr->qp_access_flags = ((roce_get_bit(context->byte_76_srqn_op_en, 4454 V2_QPC_BYTE_76_RRE_S)) << V2_QP_RWE_S) | 4455 ((roce_get_bit(context->byte_76_srqn_op_en, 4456 V2_QPC_BYTE_76_RWE_S)) << V2_QP_RRE_S) | 4457 ((roce_get_bit(context->byte_76_srqn_op_en, 4458 V2_QPC_BYTE_76_ATE_S)) << V2_QP_ATE_S); 4459 4460 if (hr_qp->ibqp.qp_type == IB_QPT_RC || 4461 hr_qp->ibqp.qp_type == IB_QPT_UC) { 4462 struct ib_global_route *grh = 4463 rdma_ah_retrieve_grh(&qp_attr->ah_attr); 4464 4465 rdma_ah_set_sl(&qp_attr->ah_attr, 4466 roce_get_field(context->byte_28_at_fl, 4467 V2_QPC_BYTE_28_SL_M, 4468 V2_QPC_BYTE_28_SL_S)); 4469 grh->flow_label = roce_get_field(context->byte_28_at_fl, 4470 V2_QPC_BYTE_28_FL_M, 4471 V2_QPC_BYTE_28_FL_S); 4472 grh->sgid_index = roce_get_field(context->byte_20_smac_sgid_idx, 4473 V2_QPC_BYTE_20_SGID_IDX_M, 4474 V2_QPC_BYTE_20_SGID_IDX_S); 4475 grh->hop_limit = roce_get_field(context->byte_24_mtu_tc, 4476 V2_QPC_BYTE_24_HOP_LIMIT_M, 4477 V2_QPC_BYTE_24_HOP_LIMIT_S); 4478 grh->traffic_class = roce_get_field(context->byte_24_mtu_tc, 4479 V2_QPC_BYTE_24_TC_M, 4480 V2_QPC_BYTE_24_TC_S); 4481 4482 memcpy(grh->dgid.raw, context->dgid, sizeof(grh->dgid.raw)); 4483 } 4484 4485 qp_attr->port_num = hr_qp->port + 1; 4486 qp_attr->sq_draining = 0; 4487 qp_attr->max_rd_atomic = 1 << roce_get_field(context->byte_208_irrl, 4488 V2_QPC_BYTE_208_SR_MAX_M, 4489 V2_QPC_BYTE_208_SR_MAX_S); 4490 qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context->byte_140_raq, 4491 V2_QPC_BYTE_140_RR_MAX_M, 4492 V2_QPC_BYTE_140_RR_MAX_S); 4493 qp_attr->min_rnr_timer = (u8)roce_get_field(context->byte_80_rnr_rx_cqn, 4494 V2_QPC_BYTE_80_MIN_RNR_TIME_M, 4495 V2_QPC_BYTE_80_MIN_RNR_TIME_S); 4496 qp_attr->timeout = (u8)roce_get_field(context->byte_28_at_fl, 4497 V2_QPC_BYTE_28_AT_M, 4498 V2_QPC_BYTE_28_AT_S); 4499 qp_attr->retry_cnt = roce_get_field(context->byte_212_lsn, 4500 V2_QPC_BYTE_212_RETRY_CNT_M, 4501 V2_QPC_BYTE_212_RETRY_CNT_S); 4502 qp_attr->rnr_retry = context->rq_rnr_timer; 4503 4504 done: 4505 qp_attr->cur_qp_state = qp_attr->qp_state; 4506 qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt; 4507 qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs; 4508 4509 if (!ibqp->uobject) { 4510 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt; 4511 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs; 4512 } else { 4513 qp_attr->cap.max_send_wr = 0; 4514 qp_attr->cap.max_send_sge = 0; 4515 } 4516 4517 qp_init_attr->cap = qp_attr->cap; 4518 4519 out: 4520 mutex_unlock(&hr_qp->mutex); 4521 kfree(context); 4522 return ret; 4523 } 4524 4525 static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, 4526 struct hns_roce_qp *hr_qp, 4527 struct ib_udata *udata) 4528 { 4529 struct hns_roce_cq *send_cq, *recv_cq; 4530 struct device *dev = hr_dev->dev; 4531 int ret; 4532 4533 if (hr_qp->ibqp.qp_type == IB_QPT_RC && hr_qp->state != IB_QPS_RESET) { 4534 /* Modify qp to reset before destroying qp */ 4535 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0, 4536 hr_qp->state, IB_QPS_RESET); 4537 if (ret) { 4538 dev_err(dev, "modify QP %06lx to ERR failed.\n", 4539 hr_qp->qpn); 4540 return ret; 4541 } 4542 } 4543 4544 send_cq = to_hr_cq(hr_qp->ibqp.send_cq); 4545 recv_cq = to_hr_cq(hr_qp->ibqp.recv_cq); 4546 4547 hns_roce_lock_cqs(send_cq, recv_cq); 4548 4549 if (!udata) { 4550 __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ? 4551 to_hr_srq(hr_qp->ibqp.srq) : NULL); 4552 if (send_cq != recv_cq) 4553 __hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL); 4554 } 4555 4556 hns_roce_qp_remove(hr_dev, hr_qp); 4557 4558 hns_roce_unlock_cqs(send_cq, recv_cq); 4559 4560 hns_roce_qp_free(hr_dev, hr_qp); 4561 4562 /* Not special_QP, free their QPN */ 4563 if ((hr_qp->ibqp.qp_type == IB_QPT_RC) || 4564 (hr_qp->ibqp.qp_type == IB_QPT_UC) || 4565 (hr_qp->ibqp.qp_type == IB_QPT_UD)) 4566 hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1); 4567 4568 hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr); 4569 4570 if (udata) { 4571 struct hns_roce_ucontext *context = 4572 rdma_udata_to_drv_context( 4573 udata, 4574 struct hns_roce_ucontext, 4575 ibucontext); 4576 4577 if (hr_qp->sq.wqe_cnt && (hr_qp->sdb_en == 1)) 4578 hns_roce_db_unmap_user(context, &hr_qp->sdb); 4579 4580 if (hr_qp->rq.wqe_cnt && (hr_qp->rdb_en == 1)) 4581 hns_roce_db_unmap_user(context, &hr_qp->rdb); 4582 } else { 4583 kfree(hr_qp->sq.wrid); 4584 kfree(hr_qp->rq.wrid); 4585 hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf); 4586 if (hr_qp->rq.wqe_cnt) 4587 hns_roce_free_db(hr_dev, &hr_qp->rdb); 4588 } 4589 ib_umem_release(hr_qp->umem); 4590 4591 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) && 4592 hr_qp->rq.wqe_cnt) { 4593 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list); 4594 kfree(hr_qp->rq_inl_buf.wqe_list); 4595 } 4596 4597 return 0; 4598 } 4599 4600 static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 4601 { 4602 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4603 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4604 int ret; 4605 4606 ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata); 4607 if (ret) { 4608 dev_err(hr_dev->dev, "Destroy qp failed(%d)\n", ret); 4609 return ret; 4610 } 4611 4612 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) 4613 kfree(hr_to_hr_sqp(hr_qp)); 4614 else 4615 kfree(hr_qp); 4616 4617 return 0; 4618 } 4619 4620 static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev, 4621 struct hns_roce_qp *hr_qp) 4622 { 4623 struct hns_roce_sccc_clr_done *resp; 4624 struct hns_roce_sccc_clr *clr; 4625 struct hns_roce_cmq_desc desc; 4626 int ret, i; 4627 4628 mutex_lock(&hr_dev->qp_table.scc_mutex); 4629 4630 /* set scc ctx clear done flag */ 4631 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCCC, false); 4632 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 4633 if (ret) { 4634 dev_err(hr_dev->dev, "Reset SCC ctx failed(%d)\n", ret); 4635 goto out; 4636 } 4637 4638 /* clear scc context */ 4639 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLR_SCCC, false); 4640 clr = (struct hns_roce_sccc_clr *)desc.data; 4641 clr->qpn = cpu_to_le32(hr_qp->qpn); 4642 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 4643 if (ret) { 4644 dev_err(hr_dev->dev, "Clear SCC ctx failed(%d)\n", ret); 4645 goto out; 4646 } 4647 4648 /* query scc context clear is done or not */ 4649 resp = (struct hns_roce_sccc_clr_done *)desc.data; 4650 for (i = 0; i <= HNS_ROCE_CMQ_SCC_CLR_DONE_CNT; i++) { 4651 hns_roce_cmq_setup_basic_desc(&desc, 4652 HNS_ROCE_OPC_QUERY_SCCC, true); 4653 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 4654 if (ret) { 4655 dev_err(hr_dev->dev, "Query clr cmq failed(%d)\n", ret); 4656 goto out; 4657 } 4658 4659 if (resp->clr_done) 4660 goto out; 4661 4662 msleep(20); 4663 } 4664 4665 dev_err(hr_dev->dev, "Query SCC clr done flag overtime.\n"); 4666 ret = -ETIMEDOUT; 4667 4668 out: 4669 mutex_unlock(&hr_dev->qp_table.scc_mutex); 4670 return ret; 4671 } 4672 4673 static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) 4674 { 4675 struct hns_roce_dev *hr_dev = to_hr_dev(cq->device); 4676 struct hns_roce_v2_cq_context *cq_context; 4677 struct hns_roce_cq *hr_cq = to_hr_cq(cq); 4678 struct hns_roce_v2_cq_context *cqc_mask; 4679 struct hns_roce_cmd_mailbox *mailbox; 4680 int ret; 4681 4682 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 4683 if (IS_ERR(mailbox)) 4684 return PTR_ERR(mailbox); 4685 4686 cq_context = mailbox->buf; 4687 cqc_mask = (struct hns_roce_v2_cq_context *)mailbox->buf + 1; 4688 4689 memset(cqc_mask, 0xff, sizeof(*cqc_mask)); 4690 4691 roce_set_field(cq_context->byte_56_cqe_period_maxcnt, 4692 V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S, 4693 cq_count); 4694 roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt, 4695 V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S, 4696 0); 4697 roce_set_field(cq_context->byte_56_cqe_period_maxcnt, 4698 V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S, 4699 cq_period); 4700 roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt, 4701 V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S, 4702 0); 4703 4704 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_cq->cqn, 1, 4705 HNS_ROCE_CMD_MODIFY_CQC, 4706 HNS_ROCE_CMD_TIMEOUT_MSECS); 4707 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 4708 if (ret) 4709 dev_err(hr_dev->dev, "MODIFY CQ Failed to cmd mailbox.\n"); 4710 4711 return ret; 4712 } 4713 4714 static void hns_roce_set_qps_to_err(struct hns_roce_dev *hr_dev, u32 qpn) 4715 { 4716 struct hns_roce_qp *hr_qp; 4717 struct ib_qp_attr attr; 4718 int attr_mask; 4719 int ret; 4720 4721 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn); 4722 if (!hr_qp) { 4723 dev_warn(hr_dev->dev, "no hr_qp can be found!\n"); 4724 return; 4725 } 4726 4727 if (hr_qp->ibqp.uobject) { 4728 if (hr_qp->sdb_en == 1) { 4729 hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr); 4730 if (hr_qp->rdb_en == 1) 4731 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr); 4732 } else { 4733 dev_warn(hr_dev->dev, "flush cqe is unsupported in userspace!\n"); 4734 return; 4735 } 4736 } 4737 4738 attr_mask = IB_QP_STATE; 4739 attr.qp_state = IB_QPS_ERR; 4740 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, &attr, attr_mask, 4741 hr_qp->state, IB_QPS_ERR); 4742 if (ret) 4743 dev_err(hr_dev->dev, "failed to modify qp %d to err state.\n", 4744 qpn); 4745 } 4746 4747 static void hns_roce_irq_work_handle(struct work_struct *work) 4748 { 4749 struct hns_roce_work *irq_work = 4750 container_of(work, struct hns_roce_work, work); 4751 struct device *dev = irq_work->hr_dev->dev; 4752 u32 qpn = irq_work->qpn; 4753 u32 cqn = irq_work->cqn; 4754 4755 switch (irq_work->event_type) { 4756 case HNS_ROCE_EVENT_TYPE_PATH_MIG: 4757 dev_info(dev, "Path migrated succeeded.\n"); 4758 break; 4759 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: 4760 dev_warn(dev, "Path migration failed.\n"); 4761 break; 4762 case HNS_ROCE_EVENT_TYPE_COMM_EST: 4763 break; 4764 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: 4765 dev_warn(dev, "Send queue drained.\n"); 4766 break; 4767 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: 4768 dev_err(dev, "Local work queue 0x%x catas error, sub_type:%d\n", 4769 qpn, irq_work->sub_type); 4770 hns_roce_set_qps_to_err(irq_work->hr_dev, qpn); 4771 break; 4772 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: 4773 dev_err(dev, "Invalid request local work queue 0x%x error.\n", 4774 qpn); 4775 hns_roce_set_qps_to_err(irq_work->hr_dev, qpn); 4776 break; 4777 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: 4778 dev_err(dev, "Local access violation work queue 0x%x error, sub_type:%d\n", 4779 qpn, irq_work->sub_type); 4780 hns_roce_set_qps_to_err(irq_work->hr_dev, qpn); 4781 break; 4782 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: 4783 dev_warn(dev, "SRQ limit reach.\n"); 4784 break; 4785 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: 4786 dev_warn(dev, "SRQ last wqe reach.\n"); 4787 break; 4788 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: 4789 dev_err(dev, "SRQ catas error.\n"); 4790 break; 4791 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: 4792 dev_err(dev, "CQ 0x%x access err.\n", cqn); 4793 break; 4794 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: 4795 dev_warn(dev, "CQ 0x%x overflow\n", cqn); 4796 break; 4797 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW: 4798 dev_warn(dev, "DB overflow.\n"); 4799 break; 4800 case HNS_ROCE_EVENT_TYPE_FLR: 4801 dev_warn(dev, "Function level reset.\n"); 4802 break; 4803 default: 4804 break; 4805 } 4806 4807 kfree(irq_work); 4808 } 4809 4810 static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev, 4811 struct hns_roce_eq *eq, 4812 u32 qpn, u32 cqn) 4813 { 4814 struct hns_roce_work *irq_work; 4815 4816 irq_work = kzalloc(sizeof(struct hns_roce_work), GFP_ATOMIC); 4817 if (!irq_work) 4818 return; 4819 4820 INIT_WORK(&(irq_work->work), hns_roce_irq_work_handle); 4821 irq_work->hr_dev = hr_dev; 4822 irq_work->qpn = qpn; 4823 irq_work->cqn = cqn; 4824 irq_work->event_type = eq->event_type; 4825 irq_work->sub_type = eq->sub_type; 4826 queue_work(hr_dev->irq_workq, &(irq_work->work)); 4827 } 4828 4829 static void set_eq_cons_index_v2(struct hns_roce_eq *eq) 4830 { 4831 struct hns_roce_dev *hr_dev = eq->hr_dev; 4832 u32 doorbell[2]; 4833 4834 doorbell[0] = 0; 4835 doorbell[1] = 0; 4836 4837 if (eq->type_flag == HNS_ROCE_AEQ) { 4838 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M, 4839 HNS_ROCE_V2_EQ_DB_CMD_S, 4840 eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ? 4841 HNS_ROCE_EQ_DB_CMD_AEQ : 4842 HNS_ROCE_EQ_DB_CMD_AEQ_ARMED); 4843 } else { 4844 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_TAG_M, 4845 HNS_ROCE_V2_EQ_DB_TAG_S, eq->eqn); 4846 4847 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M, 4848 HNS_ROCE_V2_EQ_DB_CMD_S, 4849 eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ? 4850 HNS_ROCE_EQ_DB_CMD_CEQ : 4851 HNS_ROCE_EQ_DB_CMD_CEQ_ARMED); 4852 } 4853 4854 roce_set_field(doorbell[1], HNS_ROCE_V2_EQ_DB_PARA_M, 4855 HNS_ROCE_V2_EQ_DB_PARA_S, 4856 (eq->cons_index & HNS_ROCE_V2_CONS_IDX_M)); 4857 4858 hns_roce_write64(hr_dev, doorbell, eq->doorbell); 4859 } 4860 4861 static struct hns_roce_aeqe *get_aeqe_v2(struct hns_roce_eq *eq, u32 entry) 4862 { 4863 u32 buf_chk_sz; 4864 unsigned long off; 4865 4866 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT); 4867 off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE; 4868 4869 return (struct hns_roce_aeqe *)((char *)(eq->buf_list->buf) + 4870 off % buf_chk_sz); 4871 } 4872 4873 static struct hns_roce_aeqe *mhop_get_aeqe(struct hns_roce_eq *eq, u32 entry) 4874 { 4875 u32 buf_chk_sz; 4876 unsigned long off; 4877 4878 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT); 4879 4880 off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE; 4881 4882 if (eq->hop_num == HNS_ROCE_HOP_NUM_0) 4883 return (struct hns_roce_aeqe *)((u8 *)(eq->bt_l0) + 4884 off % buf_chk_sz); 4885 else 4886 return (struct hns_roce_aeqe *)((u8 *) 4887 (eq->buf[off / buf_chk_sz]) + off % buf_chk_sz); 4888 } 4889 4890 static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq) 4891 { 4892 struct hns_roce_aeqe *aeqe; 4893 4894 if (!eq->hop_num) 4895 aeqe = get_aeqe_v2(eq, eq->cons_index); 4896 else 4897 aeqe = mhop_get_aeqe(eq, eq->cons_index); 4898 4899 return (roce_get_bit(aeqe->asyn, HNS_ROCE_V2_AEQ_AEQE_OWNER_S) ^ 4900 !!(eq->cons_index & eq->entries)) ? aeqe : NULL; 4901 } 4902 4903 static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev, 4904 struct hns_roce_eq *eq) 4905 { 4906 struct device *dev = hr_dev->dev; 4907 struct hns_roce_aeqe *aeqe; 4908 int aeqe_found = 0; 4909 int event_type; 4910 int sub_type; 4911 u32 srqn; 4912 u32 qpn; 4913 u32 cqn; 4914 4915 while ((aeqe = next_aeqe_sw_v2(eq))) { 4916 4917 /* Make sure we read AEQ entry after we have checked the 4918 * ownership bit 4919 */ 4920 dma_rmb(); 4921 4922 event_type = roce_get_field(aeqe->asyn, 4923 HNS_ROCE_V2_AEQE_EVENT_TYPE_M, 4924 HNS_ROCE_V2_AEQE_EVENT_TYPE_S); 4925 sub_type = roce_get_field(aeqe->asyn, 4926 HNS_ROCE_V2_AEQE_SUB_TYPE_M, 4927 HNS_ROCE_V2_AEQE_SUB_TYPE_S); 4928 qpn = roce_get_field(aeqe->event.qp_event.qp, 4929 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M, 4930 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S); 4931 cqn = roce_get_field(aeqe->event.cq_event.cq, 4932 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M, 4933 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S); 4934 srqn = roce_get_field(aeqe->event.srq_event.srq, 4935 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M, 4936 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S); 4937 4938 switch (event_type) { 4939 case HNS_ROCE_EVENT_TYPE_PATH_MIG: 4940 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: 4941 case HNS_ROCE_EVENT_TYPE_COMM_EST: 4942 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: 4943 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: 4944 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: 4945 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: 4946 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: 4947 hns_roce_qp_event(hr_dev, qpn, event_type); 4948 break; 4949 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: 4950 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: 4951 hns_roce_srq_event(hr_dev, srqn, event_type); 4952 break; 4953 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: 4954 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: 4955 hns_roce_cq_event(hr_dev, cqn, event_type); 4956 break; 4957 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW: 4958 break; 4959 case HNS_ROCE_EVENT_TYPE_MB: 4960 hns_roce_cmd_event(hr_dev, 4961 le16_to_cpu(aeqe->event.cmd.token), 4962 aeqe->event.cmd.status, 4963 le64_to_cpu(aeqe->event.cmd.out_param)); 4964 break; 4965 case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW: 4966 break; 4967 case HNS_ROCE_EVENT_TYPE_FLR: 4968 break; 4969 default: 4970 dev_err(dev, "Unhandled event %d on EQ %d at idx %u.\n", 4971 event_type, eq->eqn, eq->cons_index); 4972 break; 4973 } 4974 4975 eq->event_type = event_type; 4976 eq->sub_type = sub_type; 4977 ++eq->cons_index; 4978 aeqe_found = 1; 4979 4980 if (eq->cons_index > (2 * eq->entries - 1)) { 4981 dev_warn(dev, "cons_index overflow, set back to 0.\n"); 4982 eq->cons_index = 0; 4983 } 4984 hns_roce_v2_init_irq_work(hr_dev, eq, qpn, cqn); 4985 } 4986 4987 set_eq_cons_index_v2(eq); 4988 return aeqe_found; 4989 } 4990 4991 static struct hns_roce_ceqe *get_ceqe_v2(struct hns_roce_eq *eq, u32 entry) 4992 { 4993 u32 buf_chk_sz; 4994 unsigned long off; 4995 4996 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT); 4997 off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE; 4998 4999 return (struct hns_roce_ceqe *)((char *)(eq->buf_list->buf) + 5000 off % buf_chk_sz); 5001 } 5002 5003 static struct hns_roce_ceqe *mhop_get_ceqe(struct hns_roce_eq *eq, u32 entry) 5004 { 5005 u32 buf_chk_sz; 5006 unsigned long off; 5007 5008 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT); 5009 5010 off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE; 5011 5012 if (eq->hop_num == HNS_ROCE_HOP_NUM_0) 5013 return (struct hns_roce_ceqe *)((u8 *)(eq->bt_l0) + 5014 off % buf_chk_sz); 5015 else 5016 return (struct hns_roce_ceqe *)((u8 *)(eq->buf[off / 5017 buf_chk_sz]) + off % buf_chk_sz); 5018 } 5019 5020 static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq) 5021 { 5022 struct hns_roce_ceqe *ceqe; 5023 5024 if (!eq->hop_num) 5025 ceqe = get_ceqe_v2(eq, eq->cons_index); 5026 else 5027 ceqe = mhop_get_ceqe(eq, eq->cons_index); 5028 5029 return (!!(roce_get_bit(ceqe->comp, HNS_ROCE_V2_CEQ_CEQE_OWNER_S))) ^ 5030 (!!(eq->cons_index & eq->entries)) ? ceqe : NULL; 5031 } 5032 5033 static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev, 5034 struct hns_roce_eq *eq) 5035 { 5036 struct device *dev = hr_dev->dev; 5037 struct hns_roce_ceqe *ceqe; 5038 int ceqe_found = 0; 5039 u32 cqn; 5040 5041 while ((ceqe = next_ceqe_sw_v2(eq))) { 5042 5043 /* Make sure we read CEQ entry after we have checked the 5044 * ownership bit 5045 */ 5046 dma_rmb(); 5047 5048 cqn = roce_get_field(ceqe->comp, 5049 HNS_ROCE_V2_CEQE_COMP_CQN_M, 5050 HNS_ROCE_V2_CEQE_COMP_CQN_S); 5051 5052 hns_roce_cq_completion(hr_dev, cqn); 5053 5054 ++eq->cons_index; 5055 ceqe_found = 1; 5056 5057 if (eq->cons_index > (2 * eq->entries - 1)) { 5058 dev_warn(dev, "cons_index overflow, set back to 0.\n"); 5059 eq->cons_index = 0; 5060 } 5061 } 5062 5063 set_eq_cons_index_v2(eq); 5064 5065 return ceqe_found; 5066 } 5067 5068 static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr) 5069 { 5070 struct hns_roce_eq *eq = eq_ptr; 5071 struct hns_roce_dev *hr_dev = eq->hr_dev; 5072 int int_work = 0; 5073 5074 if (eq->type_flag == HNS_ROCE_CEQ) 5075 /* Completion event interrupt */ 5076 int_work = hns_roce_v2_ceq_int(hr_dev, eq); 5077 else 5078 /* Asychronous event interrupt */ 5079 int_work = hns_roce_v2_aeq_int(hr_dev, eq); 5080 5081 return IRQ_RETVAL(int_work); 5082 } 5083 5084 static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id) 5085 { 5086 struct hns_roce_dev *hr_dev = dev_id; 5087 struct device *dev = hr_dev->dev; 5088 int int_work = 0; 5089 u32 int_st; 5090 u32 int_en; 5091 5092 /* Abnormal interrupt */ 5093 int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG); 5094 int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG); 5095 5096 if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) { 5097 struct pci_dev *pdev = hr_dev->pci_dev; 5098 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); 5099 const struct hnae3_ae_ops *ops = ae_dev->ops; 5100 5101 dev_err(dev, "AEQ overflow!\n"); 5102 5103 roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S, 1); 5104 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st); 5105 5106 /* Set reset level for reset_event() */ 5107 if (ops->set_default_reset_request) 5108 ops->set_default_reset_request(ae_dev, 5109 HNAE3_FUNC_RESET); 5110 if (ops->reset_event) 5111 ops->reset_event(pdev, NULL); 5112 5113 roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1); 5114 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); 5115 5116 int_work = 1; 5117 } else if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S)) { 5118 dev_err(dev, "BUS ERR!\n"); 5119 5120 roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S, 1); 5121 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st); 5122 5123 roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1); 5124 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); 5125 5126 int_work = 1; 5127 } else if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S)) { 5128 dev_err(dev, "OTHER ERR!\n"); 5129 5130 roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S, 1); 5131 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st); 5132 5133 roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1); 5134 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); 5135 5136 int_work = 1; 5137 } else 5138 dev_err(dev, "There is no abnormal irq found!\n"); 5139 5140 return IRQ_RETVAL(int_work); 5141 } 5142 5143 static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev, 5144 int eq_num, int enable_flag) 5145 { 5146 int i; 5147 5148 if (enable_flag == EQ_ENABLE) { 5149 for (i = 0; i < eq_num; i++) 5150 roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG + 5151 i * EQ_REG_OFFSET, 5152 HNS_ROCE_V2_VF_EVENT_INT_EN_M); 5153 5154 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, 5155 HNS_ROCE_V2_VF_ABN_INT_EN_M); 5156 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, 5157 HNS_ROCE_V2_VF_ABN_INT_CFG_M); 5158 } else { 5159 for (i = 0; i < eq_num; i++) 5160 roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG + 5161 i * EQ_REG_OFFSET, 5162 HNS_ROCE_V2_VF_EVENT_INT_EN_M & 0x0); 5163 5164 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, 5165 HNS_ROCE_V2_VF_ABN_INT_EN_M & 0x0); 5166 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, 5167 HNS_ROCE_V2_VF_ABN_INT_CFG_M & 0x0); 5168 } 5169 } 5170 5171 static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn) 5172 { 5173 struct device *dev = hr_dev->dev; 5174 int ret; 5175 5176 if (eqn < hr_dev->caps.num_comp_vectors) 5177 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M, 5178 0, HNS_ROCE_CMD_DESTROY_CEQC, 5179 HNS_ROCE_CMD_TIMEOUT_MSECS); 5180 else 5181 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M, 5182 0, HNS_ROCE_CMD_DESTROY_AEQC, 5183 HNS_ROCE_CMD_TIMEOUT_MSECS); 5184 if (ret) 5185 dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn); 5186 } 5187 5188 static void hns_roce_mhop_free_eq(struct hns_roce_dev *hr_dev, 5189 struct hns_roce_eq *eq) 5190 { 5191 struct device *dev = hr_dev->dev; 5192 u64 idx; 5193 u64 size; 5194 u32 buf_chk_sz; 5195 u32 bt_chk_sz; 5196 u32 mhop_num; 5197 int eqe_alloc; 5198 int i = 0; 5199 int j = 0; 5200 5201 mhop_num = hr_dev->caps.eqe_hop_num; 5202 buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT); 5203 bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT); 5204 5205 /* hop_num = 0 */ 5206 if (mhop_num == HNS_ROCE_HOP_NUM_0) { 5207 dma_free_coherent(dev, (unsigned int)(eq->entries * 5208 eq->eqe_size), eq->bt_l0, eq->l0_dma); 5209 return; 5210 } 5211 5212 /* hop_num = 1 or hop = 2 */ 5213 dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma); 5214 if (mhop_num == 1) { 5215 for (i = 0; i < eq->l0_last_num; i++) { 5216 if (i == eq->l0_last_num - 1) { 5217 eqe_alloc = i * (buf_chk_sz / eq->eqe_size); 5218 size = (eq->entries - eqe_alloc) * eq->eqe_size; 5219 dma_free_coherent(dev, size, eq->buf[i], 5220 eq->buf_dma[i]); 5221 break; 5222 } 5223 dma_free_coherent(dev, buf_chk_sz, eq->buf[i], 5224 eq->buf_dma[i]); 5225 } 5226 } else if (mhop_num == 2) { 5227 for (i = 0; i < eq->l0_last_num; i++) { 5228 dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i], 5229 eq->l1_dma[i]); 5230 5231 for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) { 5232 idx = i * (bt_chk_sz / BA_BYTE_LEN) + j; 5233 if ((i == eq->l0_last_num - 1) 5234 && j == eq->l1_last_num - 1) { 5235 eqe_alloc = (buf_chk_sz / eq->eqe_size) 5236 * idx; 5237 size = (eq->entries - eqe_alloc) 5238 * eq->eqe_size; 5239 dma_free_coherent(dev, size, 5240 eq->buf[idx], 5241 eq->buf_dma[idx]); 5242 break; 5243 } 5244 dma_free_coherent(dev, buf_chk_sz, eq->buf[idx], 5245 eq->buf_dma[idx]); 5246 } 5247 } 5248 } 5249 kfree(eq->buf_dma); 5250 kfree(eq->buf); 5251 kfree(eq->l1_dma); 5252 kfree(eq->bt_l1); 5253 eq->buf_dma = NULL; 5254 eq->buf = NULL; 5255 eq->l1_dma = NULL; 5256 eq->bt_l1 = NULL; 5257 } 5258 5259 static void hns_roce_v2_free_eq(struct hns_roce_dev *hr_dev, 5260 struct hns_roce_eq *eq) 5261 { 5262 u32 buf_chk_sz; 5263 5264 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT); 5265 5266 if (hr_dev->caps.eqe_hop_num) { 5267 hns_roce_mhop_free_eq(hr_dev, eq); 5268 return; 5269 } 5270 5271 if (eq->buf_list) 5272 dma_free_coherent(hr_dev->dev, buf_chk_sz, 5273 eq->buf_list->buf, eq->buf_list->map); 5274 } 5275 5276 static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev, 5277 struct hns_roce_eq *eq, 5278 void *mb_buf) 5279 { 5280 struct hns_roce_eq_context *eqc; 5281 5282 eqc = mb_buf; 5283 memset(eqc, 0, sizeof(struct hns_roce_eq_context)); 5284 5285 /* init eqc */ 5286 eq->doorbell = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG; 5287 eq->hop_num = hr_dev->caps.eqe_hop_num; 5288 eq->cons_index = 0; 5289 eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0; 5290 eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0; 5291 eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED; 5292 eq->eqe_ba_pg_sz = hr_dev->caps.eqe_ba_pg_sz; 5293 eq->eqe_buf_pg_sz = hr_dev->caps.eqe_buf_pg_sz; 5294 eq->shift = ilog2((unsigned int)eq->entries); 5295 5296 if (!eq->hop_num) 5297 eq->eqe_ba = eq->buf_list->map; 5298 else 5299 eq->eqe_ba = eq->l0_dma; 5300 5301 /* set eqc state */ 5302 roce_set_field(eqc->byte_4, 5303 HNS_ROCE_EQC_EQ_ST_M, 5304 HNS_ROCE_EQC_EQ_ST_S, 5305 HNS_ROCE_V2_EQ_STATE_VALID); 5306 5307 /* set eqe hop num */ 5308 roce_set_field(eqc->byte_4, 5309 HNS_ROCE_EQC_HOP_NUM_M, 5310 HNS_ROCE_EQC_HOP_NUM_S, eq->hop_num); 5311 5312 /* set eqc over_ignore */ 5313 roce_set_field(eqc->byte_4, 5314 HNS_ROCE_EQC_OVER_IGNORE_M, 5315 HNS_ROCE_EQC_OVER_IGNORE_S, eq->over_ignore); 5316 5317 /* set eqc coalesce */ 5318 roce_set_field(eqc->byte_4, 5319 HNS_ROCE_EQC_COALESCE_M, 5320 HNS_ROCE_EQC_COALESCE_S, eq->coalesce); 5321 5322 /* set eqc arm_state */ 5323 roce_set_field(eqc->byte_4, 5324 HNS_ROCE_EQC_ARM_ST_M, 5325 HNS_ROCE_EQC_ARM_ST_S, eq->arm_st); 5326 5327 /* set eqn */ 5328 roce_set_field(eqc->byte_4, 5329 HNS_ROCE_EQC_EQN_M, 5330 HNS_ROCE_EQC_EQN_S, eq->eqn); 5331 5332 /* set eqe_cnt */ 5333 roce_set_field(eqc->byte_4, 5334 HNS_ROCE_EQC_EQE_CNT_M, 5335 HNS_ROCE_EQC_EQE_CNT_S, 5336 HNS_ROCE_EQ_INIT_EQE_CNT); 5337 5338 /* set eqe_ba_pg_sz */ 5339 roce_set_field(eqc->byte_8, 5340 HNS_ROCE_EQC_BA_PG_SZ_M, 5341 HNS_ROCE_EQC_BA_PG_SZ_S, 5342 eq->eqe_ba_pg_sz + PG_SHIFT_OFFSET); 5343 5344 /* set eqe_buf_pg_sz */ 5345 roce_set_field(eqc->byte_8, 5346 HNS_ROCE_EQC_BUF_PG_SZ_M, 5347 HNS_ROCE_EQC_BUF_PG_SZ_S, 5348 eq->eqe_buf_pg_sz + PG_SHIFT_OFFSET); 5349 5350 /* set eq_producer_idx */ 5351 roce_set_field(eqc->byte_8, 5352 HNS_ROCE_EQC_PROD_INDX_M, 5353 HNS_ROCE_EQC_PROD_INDX_S, 5354 HNS_ROCE_EQ_INIT_PROD_IDX); 5355 5356 /* set eq_max_cnt */ 5357 roce_set_field(eqc->byte_12, 5358 HNS_ROCE_EQC_MAX_CNT_M, 5359 HNS_ROCE_EQC_MAX_CNT_S, eq->eq_max_cnt); 5360 5361 /* set eq_period */ 5362 roce_set_field(eqc->byte_12, 5363 HNS_ROCE_EQC_PERIOD_M, 5364 HNS_ROCE_EQC_PERIOD_S, eq->eq_period); 5365 5366 /* set eqe_report_timer */ 5367 roce_set_field(eqc->eqe_report_timer, 5368 HNS_ROCE_EQC_REPORT_TIMER_M, 5369 HNS_ROCE_EQC_REPORT_TIMER_S, 5370 HNS_ROCE_EQ_INIT_REPORT_TIMER); 5371 5372 /* set eqe_ba [34:3] */ 5373 roce_set_field(eqc->eqe_ba0, 5374 HNS_ROCE_EQC_EQE_BA_L_M, 5375 HNS_ROCE_EQC_EQE_BA_L_S, eq->eqe_ba >> 3); 5376 5377 /* set eqe_ba [64:35] */ 5378 roce_set_field(eqc->eqe_ba1, 5379 HNS_ROCE_EQC_EQE_BA_H_M, 5380 HNS_ROCE_EQC_EQE_BA_H_S, eq->eqe_ba >> 35); 5381 5382 /* set eq shift */ 5383 roce_set_field(eqc->byte_28, 5384 HNS_ROCE_EQC_SHIFT_M, 5385 HNS_ROCE_EQC_SHIFT_S, eq->shift); 5386 5387 /* set eq MSI_IDX */ 5388 roce_set_field(eqc->byte_28, 5389 HNS_ROCE_EQC_MSI_INDX_M, 5390 HNS_ROCE_EQC_MSI_INDX_S, 5391 HNS_ROCE_EQ_INIT_MSI_IDX); 5392 5393 /* set cur_eqe_ba [27:12] */ 5394 roce_set_field(eqc->byte_28, 5395 HNS_ROCE_EQC_CUR_EQE_BA_L_M, 5396 HNS_ROCE_EQC_CUR_EQE_BA_L_S, eq->cur_eqe_ba >> 12); 5397 5398 /* set cur_eqe_ba [59:28] */ 5399 roce_set_field(eqc->byte_32, 5400 HNS_ROCE_EQC_CUR_EQE_BA_M_M, 5401 HNS_ROCE_EQC_CUR_EQE_BA_M_S, eq->cur_eqe_ba >> 28); 5402 5403 /* set cur_eqe_ba [63:60] */ 5404 roce_set_field(eqc->byte_36, 5405 HNS_ROCE_EQC_CUR_EQE_BA_H_M, 5406 HNS_ROCE_EQC_CUR_EQE_BA_H_S, eq->cur_eqe_ba >> 60); 5407 5408 /* set eq consumer idx */ 5409 roce_set_field(eqc->byte_36, 5410 HNS_ROCE_EQC_CONS_INDX_M, 5411 HNS_ROCE_EQC_CONS_INDX_S, 5412 HNS_ROCE_EQ_INIT_CONS_IDX); 5413 5414 /* set nex_eqe_ba[43:12] */ 5415 roce_set_field(eqc->nxt_eqe_ba0, 5416 HNS_ROCE_EQC_NXT_EQE_BA_L_M, 5417 HNS_ROCE_EQC_NXT_EQE_BA_L_S, eq->nxt_eqe_ba >> 12); 5418 5419 /* set nex_eqe_ba[63:44] */ 5420 roce_set_field(eqc->nxt_eqe_ba1, 5421 HNS_ROCE_EQC_NXT_EQE_BA_H_M, 5422 HNS_ROCE_EQC_NXT_EQE_BA_H_S, eq->nxt_eqe_ba >> 44); 5423 } 5424 5425 static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev, 5426 struct hns_roce_eq *eq) 5427 { 5428 struct device *dev = hr_dev->dev; 5429 int eq_alloc_done = 0; 5430 int eq_buf_cnt = 0; 5431 int eqe_alloc; 5432 u32 buf_chk_sz; 5433 u32 bt_chk_sz; 5434 u32 mhop_num; 5435 u64 size; 5436 u64 idx; 5437 int ba_num; 5438 int bt_num; 5439 int record_i; 5440 int record_j; 5441 int i = 0; 5442 int j = 0; 5443 5444 mhop_num = hr_dev->caps.eqe_hop_num; 5445 buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT); 5446 bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT); 5447 5448 ba_num = DIV_ROUND_UP(PAGE_ALIGN(eq->entries * eq->eqe_size), 5449 buf_chk_sz); 5450 bt_num = DIV_ROUND_UP(ba_num, bt_chk_sz / BA_BYTE_LEN); 5451 5452 /* hop_num = 0 */ 5453 if (mhop_num == HNS_ROCE_HOP_NUM_0) { 5454 if (eq->entries > buf_chk_sz / eq->eqe_size) { 5455 dev_err(dev, "eq entries %d is larger than buf_pg_sz!", 5456 eq->entries); 5457 return -EINVAL; 5458 } 5459 eq->bt_l0 = dma_alloc_coherent(dev, eq->entries * eq->eqe_size, 5460 &(eq->l0_dma), GFP_KERNEL); 5461 if (!eq->bt_l0) 5462 return -ENOMEM; 5463 5464 eq->cur_eqe_ba = eq->l0_dma; 5465 eq->nxt_eqe_ba = 0; 5466 5467 return 0; 5468 } 5469 5470 eq->buf_dma = kcalloc(ba_num, sizeof(*eq->buf_dma), GFP_KERNEL); 5471 if (!eq->buf_dma) 5472 return -ENOMEM; 5473 eq->buf = kcalloc(ba_num, sizeof(*eq->buf), GFP_KERNEL); 5474 if (!eq->buf) 5475 goto err_kcalloc_buf; 5476 5477 if (mhop_num == 2) { 5478 eq->l1_dma = kcalloc(bt_num, sizeof(*eq->l1_dma), GFP_KERNEL); 5479 if (!eq->l1_dma) 5480 goto err_kcalloc_l1_dma; 5481 5482 eq->bt_l1 = kcalloc(bt_num, sizeof(*eq->bt_l1), GFP_KERNEL); 5483 if (!eq->bt_l1) 5484 goto err_kcalloc_bt_l1; 5485 } 5486 5487 /* alloc L0 BT */ 5488 eq->bt_l0 = dma_alloc_coherent(dev, bt_chk_sz, &eq->l0_dma, GFP_KERNEL); 5489 if (!eq->bt_l0) 5490 goto err_dma_alloc_l0; 5491 5492 if (mhop_num == 1) { 5493 if (ba_num > (bt_chk_sz / BA_BYTE_LEN)) 5494 dev_err(dev, "ba_num %d is too large for 1 hop\n", 5495 ba_num); 5496 5497 /* alloc buf */ 5498 for (i = 0; i < bt_chk_sz / BA_BYTE_LEN; i++) { 5499 if (eq_buf_cnt + 1 < ba_num) { 5500 size = buf_chk_sz; 5501 } else { 5502 eqe_alloc = i * (buf_chk_sz / eq->eqe_size); 5503 size = (eq->entries - eqe_alloc) * eq->eqe_size; 5504 } 5505 eq->buf[i] = dma_alloc_coherent(dev, size, 5506 &(eq->buf_dma[i]), 5507 GFP_KERNEL); 5508 if (!eq->buf[i]) 5509 goto err_dma_alloc_buf; 5510 5511 *(eq->bt_l0 + i) = eq->buf_dma[i]; 5512 5513 eq_buf_cnt++; 5514 if (eq_buf_cnt >= ba_num) 5515 break; 5516 } 5517 eq->cur_eqe_ba = eq->buf_dma[0]; 5518 eq->nxt_eqe_ba = eq->buf_dma[1]; 5519 5520 } else if (mhop_num == 2) { 5521 /* alloc L1 BT and buf */ 5522 for (i = 0; i < bt_chk_sz / BA_BYTE_LEN; i++) { 5523 eq->bt_l1[i] = dma_alloc_coherent(dev, bt_chk_sz, 5524 &(eq->l1_dma[i]), 5525 GFP_KERNEL); 5526 if (!eq->bt_l1[i]) 5527 goto err_dma_alloc_l1; 5528 *(eq->bt_l0 + i) = eq->l1_dma[i]; 5529 5530 for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) { 5531 idx = i * bt_chk_sz / BA_BYTE_LEN + j; 5532 if (eq_buf_cnt + 1 < ba_num) { 5533 size = buf_chk_sz; 5534 } else { 5535 eqe_alloc = (buf_chk_sz / eq->eqe_size) 5536 * idx; 5537 size = (eq->entries - eqe_alloc) 5538 * eq->eqe_size; 5539 } 5540 eq->buf[idx] = dma_alloc_coherent(dev, size, 5541 &(eq->buf_dma[idx]), 5542 GFP_KERNEL); 5543 if (!eq->buf[idx]) 5544 goto err_dma_alloc_buf; 5545 5546 *(eq->bt_l1[i] + j) = eq->buf_dma[idx]; 5547 5548 eq_buf_cnt++; 5549 if (eq_buf_cnt >= ba_num) { 5550 eq_alloc_done = 1; 5551 break; 5552 } 5553 } 5554 5555 if (eq_alloc_done) 5556 break; 5557 } 5558 eq->cur_eqe_ba = eq->buf_dma[0]; 5559 eq->nxt_eqe_ba = eq->buf_dma[1]; 5560 } 5561 5562 eq->l0_last_num = i + 1; 5563 if (mhop_num == 2) 5564 eq->l1_last_num = j + 1; 5565 5566 return 0; 5567 5568 err_dma_alloc_l1: 5569 dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma); 5570 eq->bt_l0 = NULL; 5571 eq->l0_dma = 0; 5572 for (i -= 1; i >= 0; i--) { 5573 dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i], 5574 eq->l1_dma[i]); 5575 5576 for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) { 5577 idx = i * bt_chk_sz / BA_BYTE_LEN + j; 5578 dma_free_coherent(dev, buf_chk_sz, eq->buf[idx], 5579 eq->buf_dma[idx]); 5580 } 5581 } 5582 goto err_dma_alloc_l0; 5583 5584 err_dma_alloc_buf: 5585 dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma); 5586 eq->bt_l0 = NULL; 5587 eq->l0_dma = 0; 5588 5589 if (mhop_num == 1) 5590 for (i -= 1; i >= 0; i--) 5591 dma_free_coherent(dev, buf_chk_sz, eq->buf[i], 5592 eq->buf_dma[i]); 5593 else if (mhop_num == 2) { 5594 record_i = i; 5595 record_j = j; 5596 for (; i >= 0; i--) { 5597 dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i], 5598 eq->l1_dma[i]); 5599 5600 for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) { 5601 if (i == record_i && j >= record_j) 5602 break; 5603 5604 idx = i * bt_chk_sz / BA_BYTE_LEN + j; 5605 dma_free_coherent(dev, buf_chk_sz, 5606 eq->buf[idx], 5607 eq->buf_dma[idx]); 5608 } 5609 } 5610 } 5611 5612 err_dma_alloc_l0: 5613 kfree(eq->bt_l1); 5614 eq->bt_l1 = NULL; 5615 5616 err_kcalloc_bt_l1: 5617 kfree(eq->l1_dma); 5618 eq->l1_dma = NULL; 5619 5620 err_kcalloc_l1_dma: 5621 kfree(eq->buf); 5622 eq->buf = NULL; 5623 5624 err_kcalloc_buf: 5625 kfree(eq->buf_dma); 5626 eq->buf_dma = NULL; 5627 5628 return -ENOMEM; 5629 } 5630 5631 static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev, 5632 struct hns_roce_eq *eq, 5633 unsigned int eq_cmd) 5634 { 5635 struct device *dev = hr_dev->dev; 5636 struct hns_roce_cmd_mailbox *mailbox; 5637 u32 buf_chk_sz = 0; 5638 int ret; 5639 5640 /* Allocate mailbox memory */ 5641 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5642 if (IS_ERR(mailbox)) 5643 return PTR_ERR(mailbox); 5644 5645 if (!hr_dev->caps.eqe_hop_num) { 5646 buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT); 5647 5648 eq->buf_list = kzalloc(sizeof(struct hns_roce_buf_list), 5649 GFP_KERNEL); 5650 if (!eq->buf_list) { 5651 ret = -ENOMEM; 5652 goto free_cmd_mbox; 5653 } 5654 5655 eq->buf_list->buf = dma_alloc_coherent(dev, buf_chk_sz, 5656 &(eq->buf_list->map), 5657 GFP_KERNEL); 5658 if (!eq->buf_list->buf) { 5659 ret = -ENOMEM; 5660 goto err_alloc_buf; 5661 } 5662 5663 } else { 5664 ret = hns_roce_mhop_alloc_eq(hr_dev, eq); 5665 if (ret) { 5666 ret = -ENOMEM; 5667 goto free_cmd_mbox; 5668 } 5669 } 5670 5671 hns_roce_config_eqc(hr_dev, eq, mailbox->buf); 5672 5673 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, eq->eqn, 0, 5674 eq_cmd, HNS_ROCE_CMD_TIMEOUT_MSECS); 5675 if (ret) { 5676 dev_err(dev, "[mailbox cmd] create eqc failed.\n"); 5677 goto err_cmd_mbox; 5678 } 5679 5680 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5681 5682 return 0; 5683 5684 err_cmd_mbox: 5685 if (!hr_dev->caps.eqe_hop_num) 5686 dma_free_coherent(dev, buf_chk_sz, eq->buf_list->buf, 5687 eq->buf_list->map); 5688 else { 5689 hns_roce_mhop_free_eq(hr_dev, eq); 5690 goto free_cmd_mbox; 5691 } 5692 5693 err_alloc_buf: 5694 kfree(eq->buf_list); 5695 5696 free_cmd_mbox: 5697 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5698 5699 return ret; 5700 } 5701 5702 static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev) 5703 { 5704 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; 5705 struct device *dev = hr_dev->dev; 5706 struct hns_roce_eq *eq; 5707 unsigned int eq_cmd; 5708 int irq_num; 5709 int eq_num; 5710 int other_num; 5711 int comp_num; 5712 int aeq_num; 5713 int i, j, k; 5714 int ret; 5715 5716 other_num = hr_dev->caps.num_other_vectors; 5717 comp_num = hr_dev->caps.num_comp_vectors; 5718 aeq_num = hr_dev->caps.num_aeq_vectors; 5719 5720 eq_num = comp_num + aeq_num; 5721 irq_num = eq_num + other_num; 5722 5723 eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL); 5724 if (!eq_table->eq) 5725 return -ENOMEM; 5726 5727 for (i = 0; i < irq_num; i++) { 5728 hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN, 5729 GFP_KERNEL); 5730 if (!hr_dev->irq_names[i]) { 5731 ret = -ENOMEM; 5732 goto err_failed_kzalloc; 5733 } 5734 } 5735 5736 /* create eq */ 5737 for (j = 0; j < eq_num; j++) { 5738 eq = &eq_table->eq[j]; 5739 eq->hr_dev = hr_dev; 5740 eq->eqn = j; 5741 if (j < comp_num) { 5742 /* CEQ */ 5743 eq_cmd = HNS_ROCE_CMD_CREATE_CEQC; 5744 eq->type_flag = HNS_ROCE_CEQ; 5745 eq->entries = hr_dev->caps.ceqe_depth; 5746 eq->eqe_size = HNS_ROCE_CEQ_ENTRY_SIZE; 5747 eq->irq = hr_dev->irq[j + other_num + aeq_num]; 5748 eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM; 5749 eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL; 5750 } else { 5751 /* AEQ */ 5752 eq_cmd = HNS_ROCE_CMD_CREATE_AEQC; 5753 eq->type_flag = HNS_ROCE_AEQ; 5754 eq->entries = hr_dev->caps.aeqe_depth; 5755 eq->eqe_size = HNS_ROCE_AEQ_ENTRY_SIZE; 5756 eq->irq = hr_dev->irq[j - comp_num + other_num]; 5757 eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM; 5758 eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL; 5759 } 5760 5761 ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd); 5762 if (ret) { 5763 dev_err(dev, "eq create failed.\n"); 5764 goto err_create_eq_fail; 5765 } 5766 } 5767 5768 /* enable irq */ 5769 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE); 5770 5771 /* irq contains: abnormal + AEQ + CEQ*/ 5772 for (k = 0; k < irq_num; k++) 5773 if (k < other_num) 5774 snprintf((char *)hr_dev->irq_names[k], 5775 HNS_ROCE_INT_NAME_LEN, "hns-abn-%d", k); 5776 else if (k < (other_num + aeq_num)) 5777 snprintf((char *)hr_dev->irq_names[k], 5778 HNS_ROCE_INT_NAME_LEN, "hns-aeq-%d", 5779 k - other_num); 5780 else 5781 snprintf((char *)hr_dev->irq_names[k], 5782 HNS_ROCE_INT_NAME_LEN, "hns-ceq-%d", 5783 k - other_num - aeq_num); 5784 5785 for (k = 0; k < irq_num; k++) { 5786 if (k < other_num) 5787 ret = request_irq(hr_dev->irq[k], 5788 hns_roce_v2_msix_interrupt_abn, 5789 0, hr_dev->irq_names[k], hr_dev); 5790 5791 else if (k < (other_num + comp_num)) 5792 ret = request_irq(eq_table->eq[k - other_num].irq, 5793 hns_roce_v2_msix_interrupt_eq, 5794 0, hr_dev->irq_names[k + aeq_num], 5795 &eq_table->eq[k - other_num]); 5796 else 5797 ret = request_irq(eq_table->eq[k - other_num].irq, 5798 hns_roce_v2_msix_interrupt_eq, 5799 0, hr_dev->irq_names[k - comp_num], 5800 &eq_table->eq[k - other_num]); 5801 if (ret) { 5802 dev_err(dev, "Request irq error!\n"); 5803 goto err_request_irq_fail; 5804 } 5805 } 5806 5807 hr_dev->irq_workq = 5808 create_singlethread_workqueue("hns_roce_irq_workqueue"); 5809 if (!hr_dev->irq_workq) { 5810 dev_err(dev, "Create irq workqueue failed!\n"); 5811 ret = -ENOMEM; 5812 goto err_request_irq_fail; 5813 } 5814 5815 return 0; 5816 5817 err_request_irq_fail: 5818 for (k -= 1; k >= 0; k--) 5819 if (k < other_num) 5820 free_irq(hr_dev->irq[k], hr_dev); 5821 else 5822 free_irq(eq_table->eq[k - other_num].irq, 5823 &eq_table->eq[k - other_num]); 5824 5825 err_create_eq_fail: 5826 for (j -= 1; j >= 0; j--) 5827 hns_roce_v2_free_eq(hr_dev, &eq_table->eq[j]); 5828 5829 err_failed_kzalloc: 5830 for (i -= 1; i >= 0; i--) 5831 kfree(hr_dev->irq_names[i]); 5832 kfree(eq_table->eq); 5833 5834 return ret; 5835 } 5836 5837 static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev) 5838 { 5839 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; 5840 int irq_num; 5841 int eq_num; 5842 int i; 5843 5844 eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; 5845 irq_num = eq_num + hr_dev->caps.num_other_vectors; 5846 5847 /* Disable irq */ 5848 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE); 5849 5850 for (i = 0; i < hr_dev->caps.num_other_vectors; i++) 5851 free_irq(hr_dev->irq[i], hr_dev); 5852 5853 for (i = 0; i < eq_num; i++) { 5854 hns_roce_v2_destroy_eqc(hr_dev, i); 5855 5856 free_irq(eq_table->eq[i].irq, &eq_table->eq[i]); 5857 5858 hns_roce_v2_free_eq(hr_dev, &eq_table->eq[i]); 5859 } 5860 5861 for (i = 0; i < irq_num; i++) 5862 kfree(hr_dev->irq_names[i]); 5863 5864 kfree(eq_table->eq); 5865 5866 flush_workqueue(hr_dev->irq_workq); 5867 destroy_workqueue(hr_dev->irq_workq); 5868 } 5869 5870 static void hns_roce_v2_write_srqc(struct hns_roce_dev *hr_dev, 5871 struct hns_roce_srq *srq, u32 pdn, u16 xrcd, 5872 u32 cqn, void *mb_buf, u64 *mtts_wqe, 5873 u64 *mtts_idx, dma_addr_t dma_handle_wqe, 5874 dma_addr_t dma_handle_idx) 5875 { 5876 struct hns_roce_srq_context *srq_context; 5877 5878 srq_context = mb_buf; 5879 memset(srq_context, 0, sizeof(*srq_context)); 5880 5881 roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQ_ST_M, 5882 SRQC_BYTE_4_SRQ_ST_S, 1); 5883 5884 roce_set_field(srq_context->byte_4_srqn_srqst, 5885 SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M, 5886 SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S, 5887 (hr_dev->caps.srqwqe_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : 5888 hr_dev->caps.srqwqe_hop_num)); 5889 roce_set_field(srq_context->byte_4_srqn_srqst, 5890 SRQC_BYTE_4_SRQ_SHIFT_M, SRQC_BYTE_4_SRQ_SHIFT_S, 5891 ilog2(srq->max)); 5892 5893 roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQN_M, 5894 SRQC_BYTE_4_SRQN_S, srq->srqn); 5895 5896 roce_set_field(srq_context->byte_8_limit_wl, SRQC_BYTE_8_SRQ_LIMIT_WL_M, 5897 SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0); 5898 5899 roce_set_field(srq_context->byte_12_xrcd, SRQC_BYTE_12_SRQ_XRCD_M, 5900 SRQC_BYTE_12_SRQ_XRCD_S, xrcd); 5901 5902 srq_context->wqe_bt_ba = cpu_to_le32((u32)(dma_handle_wqe >> 3)); 5903 5904 roce_set_field(srq_context->byte_24_wqe_bt_ba, 5905 SRQC_BYTE_24_SRQ_WQE_BT_BA_M, 5906 SRQC_BYTE_24_SRQ_WQE_BT_BA_S, 5907 cpu_to_le32(dma_handle_wqe >> 35)); 5908 5909 roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_PD_M, 5910 SRQC_BYTE_28_PD_S, pdn); 5911 roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_RQWS_M, 5912 SRQC_BYTE_28_RQWS_S, srq->max_gs <= 0 ? 0 : 5913 fls(srq->max_gs - 1)); 5914 5915 srq_context->idx_bt_ba = (u32)(dma_handle_idx >> 3); 5916 srq_context->idx_bt_ba = cpu_to_le32(srq_context->idx_bt_ba); 5917 roce_set_field(srq_context->rsv_idx_bt_ba, 5918 SRQC_BYTE_36_SRQ_IDX_BT_BA_M, 5919 SRQC_BYTE_36_SRQ_IDX_BT_BA_S, 5920 cpu_to_le32(dma_handle_idx >> 35)); 5921 5922 srq_context->idx_cur_blk_addr = (u32)(mtts_idx[0] >> PAGE_ADDR_SHIFT); 5923 srq_context->idx_cur_blk_addr = 5924 cpu_to_le32(srq_context->idx_cur_blk_addr); 5925 roce_set_field(srq_context->byte_44_idxbufpgsz_addr, 5926 SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M, 5927 SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S, 5928 cpu_to_le32((mtts_idx[0]) >> (32 + PAGE_ADDR_SHIFT))); 5929 roce_set_field(srq_context->byte_44_idxbufpgsz_addr, 5930 SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M, 5931 SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S, 5932 hr_dev->caps.idx_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : 5933 hr_dev->caps.idx_hop_num); 5934 5935 roce_set_field(srq_context->byte_44_idxbufpgsz_addr, 5936 SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M, 5937 SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S, 5938 hr_dev->caps.idx_ba_pg_sz); 5939 roce_set_field(srq_context->byte_44_idxbufpgsz_addr, 5940 SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M, 5941 SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S, 5942 hr_dev->caps.idx_buf_pg_sz); 5943 5944 srq_context->idx_nxt_blk_addr = (u32)(mtts_idx[1] >> PAGE_ADDR_SHIFT); 5945 srq_context->idx_nxt_blk_addr = 5946 cpu_to_le32(srq_context->idx_nxt_blk_addr); 5947 roce_set_field(srq_context->rsv_idxnxtblkaddr, 5948 SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M, 5949 SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S, 5950 cpu_to_le32((mtts_idx[1]) >> (32 + PAGE_ADDR_SHIFT))); 5951 roce_set_field(srq_context->byte_56_xrc_cqn, 5952 SRQC_BYTE_56_SRQ_XRC_CQN_M, SRQC_BYTE_56_SRQ_XRC_CQN_S, 5953 cqn); 5954 roce_set_field(srq_context->byte_56_xrc_cqn, 5955 SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M, 5956 SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S, 5957 hr_dev->caps.srqwqe_ba_pg_sz + PG_SHIFT_OFFSET); 5958 roce_set_field(srq_context->byte_56_xrc_cqn, 5959 SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M, 5960 SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S, 5961 hr_dev->caps.srqwqe_buf_pg_sz + PG_SHIFT_OFFSET); 5962 5963 roce_set_bit(srq_context->db_record_addr_record_en, 5964 SRQC_BYTE_60_SRQ_RECORD_EN_S, 0); 5965 } 5966 5967 static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq, 5968 struct ib_srq_attr *srq_attr, 5969 enum ib_srq_attr_mask srq_attr_mask, 5970 struct ib_udata *udata) 5971 { 5972 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); 5973 struct hns_roce_srq *srq = to_hr_srq(ibsrq); 5974 struct hns_roce_srq_context *srq_context; 5975 struct hns_roce_srq_context *srqc_mask; 5976 struct hns_roce_cmd_mailbox *mailbox; 5977 int ret; 5978 5979 if (srq_attr_mask & IB_SRQ_LIMIT) { 5980 if (srq_attr->srq_limit >= srq->max) 5981 return -EINVAL; 5982 5983 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5984 if (IS_ERR(mailbox)) 5985 return PTR_ERR(mailbox); 5986 5987 srq_context = mailbox->buf; 5988 srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1; 5989 5990 memset(srqc_mask, 0xff, sizeof(*srqc_mask)); 5991 5992 roce_set_field(srq_context->byte_8_limit_wl, 5993 SRQC_BYTE_8_SRQ_LIMIT_WL_M, 5994 SRQC_BYTE_8_SRQ_LIMIT_WL_S, srq_attr->srq_limit); 5995 roce_set_field(srqc_mask->byte_8_limit_wl, 5996 SRQC_BYTE_8_SRQ_LIMIT_WL_M, 5997 SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0); 5998 5999 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, srq->srqn, 0, 6000 HNS_ROCE_CMD_MODIFY_SRQC, 6001 HNS_ROCE_CMD_TIMEOUT_MSECS); 6002 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 6003 if (ret) { 6004 dev_err(hr_dev->dev, 6005 "MODIFY SRQ Failed to cmd mailbox.\n"); 6006 return ret; 6007 } 6008 } 6009 6010 return 0; 6011 } 6012 6013 static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) 6014 { 6015 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); 6016 struct hns_roce_srq *srq = to_hr_srq(ibsrq); 6017 struct hns_roce_srq_context *srq_context; 6018 struct hns_roce_cmd_mailbox *mailbox; 6019 int limit_wl; 6020 int ret; 6021 6022 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 6023 if (IS_ERR(mailbox)) 6024 return PTR_ERR(mailbox); 6025 6026 srq_context = mailbox->buf; 6027 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, srq->srqn, 0, 6028 HNS_ROCE_CMD_QUERY_SRQC, 6029 HNS_ROCE_CMD_TIMEOUT_MSECS); 6030 if (ret) { 6031 dev_err(hr_dev->dev, "QUERY SRQ cmd process error\n"); 6032 goto out; 6033 } 6034 6035 limit_wl = roce_get_field(srq_context->byte_8_limit_wl, 6036 SRQC_BYTE_8_SRQ_LIMIT_WL_M, 6037 SRQC_BYTE_8_SRQ_LIMIT_WL_S); 6038 6039 attr->srq_limit = limit_wl; 6040 attr->max_wr = srq->max - 1; 6041 attr->max_sge = srq->max_gs; 6042 6043 memcpy(srq_context, mailbox->buf, sizeof(*srq_context)); 6044 6045 out: 6046 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 6047 return ret; 6048 } 6049 6050 static int find_empty_entry(struct hns_roce_idx_que *idx_que, 6051 unsigned long size) 6052 { 6053 int wqe_idx; 6054 6055 if (unlikely(bitmap_full(idx_que->bitmap, size))) 6056 return -ENOSPC; 6057 6058 wqe_idx = find_first_zero_bit(idx_que->bitmap, size); 6059 6060 bitmap_set(idx_que->bitmap, wqe_idx, 1); 6061 6062 return wqe_idx; 6063 } 6064 6065 static void fill_idx_queue(struct hns_roce_idx_que *idx_que, 6066 int cur_idx, int wqe_idx) 6067 { 6068 unsigned int *addr; 6069 6070 addr = (unsigned int *)hns_roce_buf_offset(&idx_que->idx_buf, 6071 cur_idx * idx_que->entry_sz); 6072 *addr = wqe_idx; 6073 } 6074 6075 static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq, 6076 const struct ib_recv_wr *wr, 6077 const struct ib_recv_wr **bad_wr) 6078 { 6079 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); 6080 struct hns_roce_srq *srq = to_hr_srq(ibsrq); 6081 struct hns_roce_v2_wqe_data_seg *dseg; 6082 struct hns_roce_v2_db srq_db; 6083 unsigned long flags; 6084 int ret = 0; 6085 int wqe_idx; 6086 void *wqe; 6087 int nreq; 6088 int ind; 6089 int i; 6090 6091 spin_lock_irqsave(&srq->lock, flags); 6092 6093 ind = srq->head & (srq->max - 1); 6094 6095 for (nreq = 0; wr; ++nreq, wr = wr->next) { 6096 if (unlikely(wr->num_sge > srq->max_gs)) { 6097 ret = -EINVAL; 6098 *bad_wr = wr; 6099 break; 6100 } 6101 6102 if (unlikely(srq->head == srq->tail)) { 6103 ret = -ENOMEM; 6104 *bad_wr = wr; 6105 break; 6106 } 6107 6108 wqe_idx = find_empty_entry(&srq->idx_que, srq->max); 6109 if (wqe_idx < 0) { 6110 ret = -ENOMEM; 6111 *bad_wr = wr; 6112 break; 6113 } 6114 6115 fill_idx_queue(&srq->idx_que, ind, wqe_idx); 6116 wqe = get_srq_wqe(srq, wqe_idx); 6117 dseg = (struct hns_roce_v2_wqe_data_seg *)wqe; 6118 6119 for (i = 0; i < wr->num_sge; ++i) { 6120 dseg[i].len = cpu_to_le32(wr->sg_list[i].length); 6121 dseg[i].lkey = cpu_to_le32(wr->sg_list[i].lkey); 6122 dseg[i].addr = cpu_to_le64(wr->sg_list[i].addr); 6123 } 6124 6125 if (i < srq->max_gs) { 6126 dseg[i].len = 0; 6127 dseg[i].lkey = cpu_to_le32(0x100); 6128 dseg[i].addr = 0; 6129 } 6130 6131 srq->wrid[wqe_idx] = wr->wr_id; 6132 ind = (ind + 1) & (srq->max - 1); 6133 } 6134 6135 if (likely(nreq)) { 6136 srq->head += nreq; 6137 6138 /* 6139 * Make sure that descriptors are written before 6140 * doorbell record. 6141 */ 6142 wmb(); 6143 6144 srq_db.byte_4 = HNS_ROCE_V2_SRQ_DB << V2_DB_BYTE_4_CMD_S | 6145 (srq->srqn & V2_DB_BYTE_4_TAG_M); 6146 srq_db.parameter = srq->head; 6147 6148 hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l); 6149 6150 } 6151 6152 spin_unlock_irqrestore(&srq->lock, flags); 6153 6154 return ret; 6155 } 6156 6157 static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = { 6158 .query_cqc_info = hns_roce_v2_query_cqc_info, 6159 }; 6160 6161 static const struct ib_device_ops hns_roce_v2_dev_ops = { 6162 .destroy_qp = hns_roce_v2_destroy_qp, 6163 .modify_cq = hns_roce_v2_modify_cq, 6164 .poll_cq = hns_roce_v2_poll_cq, 6165 .post_recv = hns_roce_v2_post_recv, 6166 .post_send = hns_roce_v2_post_send, 6167 .query_qp = hns_roce_v2_query_qp, 6168 .req_notify_cq = hns_roce_v2_req_notify_cq, 6169 }; 6170 6171 static const struct ib_device_ops hns_roce_v2_dev_srq_ops = { 6172 .modify_srq = hns_roce_v2_modify_srq, 6173 .post_srq_recv = hns_roce_v2_post_srq_recv, 6174 .query_srq = hns_roce_v2_query_srq, 6175 }; 6176 6177 static const struct hns_roce_hw hns_roce_hw_v2 = { 6178 .cmq_init = hns_roce_v2_cmq_init, 6179 .cmq_exit = hns_roce_v2_cmq_exit, 6180 .hw_profile = hns_roce_v2_profile, 6181 .hw_init = hns_roce_v2_init, 6182 .hw_exit = hns_roce_v2_exit, 6183 .post_mbox = hns_roce_v2_post_mbox, 6184 .chk_mbox = hns_roce_v2_chk_mbox, 6185 .rst_prc_mbox = hns_roce_v2_rst_process_cmd, 6186 .set_gid = hns_roce_v2_set_gid, 6187 .set_mac = hns_roce_v2_set_mac, 6188 .write_mtpt = hns_roce_v2_write_mtpt, 6189 .rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt, 6190 .frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt, 6191 .mw_write_mtpt = hns_roce_v2_mw_write_mtpt, 6192 .write_cqc = hns_roce_v2_write_cqc, 6193 .set_hem = hns_roce_v2_set_hem, 6194 .clear_hem = hns_roce_v2_clear_hem, 6195 .modify_qp = hns_roce_v2_modify_qp, 6196 .query_qp = hns_roce_v2_query_qp, 6197 .destroy_qp = hns_roce_v2_destroy_qp, 6198 .qp_flow_control_init = hns_roce_v2_qp_flow_control_init, 6199 .modify_cq = hns_roce_v2_modify_cq, 6200 .post_send = hns_roce_v2_post_send, 6201 .post_recv = hns_roce_v2_post_recv, 6202 .req_notify_cq = hns_roce_v2_req_notify_cq, 6203 .poll_cq = hns_roce_v2_poll_cq, 6204 .init_eq = hns_roce_v2_init_eq_table, 6205 .cleanup_eq = hns_roce_v2_cleanup_eq_table, 6206 .write_srqc = hns_roce_v2_write_srqc, 6207 .modify_srq = hns_roce_v2_modify_srq, 6208 .query_srq = hns_roce_v2_query_srq, 6209 .post_srq_recv = hns_roce_v2_post_srq_recv, 6210 .hns_roce_dev_ops = &hns_roce_v2_dev_ops, 6211 .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops, 6212 }; 6213 6214 static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = { 6215 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0}, 6216 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0}, 6217 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0}, 6218 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0}, 6219 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0}, 6220 /* required last entry */ 6221 {0, } 6222 }; 6223 6224 MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl); 6225 6226 static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev, 6227 struct hnae3_handle *handle) 6228 { 6229 struct hns_roce_v2_priv *priv = hr_dev->priv; 6230 int i; 6231 6232 hr_dev->hw = &hns_roce_hw_v2; 6233 hr_dev->dfx = &hns_roce_dfx_hw_v2; 6234 hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG; 6235 hr_dev->odb_offset = hr_dev->sdb_offset; 6236 6237 /* Get info from NIC driver. */ 6238 hr_dev->reg_base = handle->rinfo.roce_io_base; 6239 hr_dev->caps.num_ports = 1; 6240 hr_dev->iboe.netdevs[0] = handle->rinfo.netdev; 6241 hr_dev->iboe.phy_port[0] = 0; 6242 6243 addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid, 6244 hr_dev->iboe.netdevs[0]->dev_addr); 6245 6246 for (i = 0; i < HNS_ROCE_V2_MAX_IRQ_NUM; i++) 6247 hr_dev->irq[i] = pci_irq_vector(handle->pdev, 6248 i + handle->rinfo.base_vector); 6249 6250 /* cmd issue mode: 0 is poll, 1 is event */ 6251 hr_dev->cmd_mod = 1; 6252 hr_dev->loop_idc = 0; 6253 6254 hr_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle); 6255 priv->handle = handle; 6256 6257 return 0; 6258 } 6259 6260 static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) 6261 { 6262 struct hns_roce_dev *hr_dev; 6263 int ret; 6264 6265 hr_dev = ib_alloc_device(hns_roce_dev, ib_dev); 6266 if (!hr_dev) 6267 return -ENOMEM; 6268 6269 hr_dev->priv = kzalloc(sizeof(struct hns_roce_v2_priv), GFP_KERNEL); 6270 if (!hr_dev->priv) { 6271 ret = -ENOMEM; 6272 goto error_failed_kzalloc; 6273 } 6274 6275 hr_dev->pci_dev = handle->pdev; 6276 hr_dev->dev = &handle->pdev->dev; 6277 6278 ret = hns_roce_hw_v2_get_cfg(hr_dev, handle); 6279 if (ret) { 6280 dev_err(hr_dev->dev, "Get Configuration failed!\n"); 6281 goto error_failed_get_cfg; 6282 } 6283 6284 ret = hns_roce_init(hr_dev); 6285 if (ret) { 6286 dev_err(hr_dev->dev, "RoCE Engine init failed!\n"); 6287 goto error_failed_get_cfg; 6288 } 6289 6290 handle->priv = hr_dev; 6291 6292 return 0; 6293 6294 error_failed_get_cfg: 6295 kfree(hr_dev->priv); 6296 6297 error_failed_kzalloc: 6298 ib_dealloc_device(&hr_dev->ib_dev); 6299 6300 return ret; 6301 } 6302 6303 static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, 6304 bool reset) 6305 { 6306 struct hns_roce_dev *hr_dev = (struct hns_roce_dev *)handle->priv; 6307 6308 if (!hr_dev) 6309 return; 6310 6311 handle->priv = NULL; 6312 hns_roce_exit(hr_dev); 6313 kfree(hr_dev->priv); 6314 ib_dealloc_device(&hr_dev->ib_dev); 6315 } 6316 6317 static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) 6318 { 6319 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 6320 const struct pci_device_id *id; 6321 struct device *dev = &handle->pdev->dev; 6322 int ret; 6323 6324 handle->rinfo.instance_state = HNS_ROCE_STATE_INIT; 6325 6326 if (ops->ae_dev_resetting(handle) || ops->get_hw_reset_stat(handle)) { 6327 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; 6328 goto reset_chk_err; 6329 } 6330 6331 id = pci_match_id(hns_roce_hw_v2_pci_tbl, handle->pdev); 6332 if (!id) 6333 return 0; 6334 6335 ret = __hns_roce_hw_v2_init_instance(handle); 6336 if (ret) { 6337 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; 6338 dev_err(dev, "RoCE instance init failed! ret = %d\n", ret); 6339 if (ops->ae_dev_resetting(handle) || 6340 ops->get_hw_reset_stat(handle)) 6341 goto reset_chk_err; 6342 else 6343 return ret; 6344 } 6345 6346 handle->rinfo.instance_state = HNS_ROCE_STATE_INITED; 6347 6348 6349 return 0; 6350 6351 reset_chk_err: 6352 dev_err(dev, "Device is busy in resetting state.\n" 6353 "please retry later.\n"); 6354 6355 return -EBUSY; 6356 } 6357 6358 static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, 6359 bool reset) 6360 { 6361 if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) 6362 return; 6363 6364 handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT; 6365 6366 __hns_roce_hw_v2_uninit_instance(handle, reset); 6367 6368 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; 6369 } 6370 static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle) 6371 { 6372 struct hns_roce_dev *hr_dev; 6373 struct ib_event event; 6374 6375 if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) { 6376 set_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state); 6377 return 0; 6378 } 6379 6380 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN; 6381 clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state); 6382 6383 hr_dev = (struct hns_roce_dev *)handle->priv; 6384 if (!hr_dev) 6385 return 0; 6386 6387 hr_dev->is_reset = true; 6388 hr_dev->active = false; 6389 hr_dev->dis_db = true; 6390 6391 event.event = IB_EVENT_DEVICE_FATAL; 6392 event.device = &hr_dev->ib_dev; 6393 event.element.port_num = 1; 6394 ib_dispatch_event(&event); 6395 6396 return 0; 6397 } 6398 6399 static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle) 6400 { 6401 struct device *dev = &handle->pdev->dev; 6402 int ret; 6403 6404 if (test_and_clear_bit(HNS_ROCE_RST_DIRECT_RETURN, 6405 &handle->rinfo.state)) { 6406 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED; 6407 return 0; 6408 } 6409 6410 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INIT; 6411 6412 dev_info(&handle->pdev->dev, "In reset process RoCE client reinit.\n"); 6413 ret = __hns_roce_hw_v2_init_instance(handle); 6414 if (ret) { 6415 /* when reset notify type is HNAE3_INIT_CLIENT In reset notify 6416 * callback function, RoCE Engine reinitialize. If RoCE reinit 6417 * failed, we should inform NIC driver. 6418 */ 6419 handle->priv = NULL; 6420 dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret); 6421 } else { 6422 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED; 6423 dev_info(dev, "Reset done, RoCE client reinit finished.\n"); 6424 } 6425 6426 return ret; 6427 } 6428 6429 static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle) 6430 { 6431 if (test_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state)) 6432 return 0; 6433 6434 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT; 6435 dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n"); 6436 msleep(100); 6437 __hns_roce_hw_v2_uninit_instance(handle, false); 6438 6439 return 0; 6440 } 6441 6442 static int hns_roce_hw_v2_reset_notify(struct hnae3_handle *handle, 6443 enum hnae3_reset_notify_type type) 6444 { 6445 int ret = 0; 6446 6447 switch (type) { 6448 case HNAE3_DOWN_CLIENT: 6449 ret = hns_roce_hw_v2_reset_notify_down(handle); 6450 break; 6451 case HNAE3_INIT_CLIENT: 6452 ret = hns_roce_hw_v2_reset_notify_init(handle); 6453 break; 6454 case HNAE3_UNINIT_CLIENT: 6455 ret = hns_roce_hw_v2_reset_notify_uninit(handle); 6456 break; 6457 default: 6458 break; 6459 } 6460 6461 return ret; 6462 } 6463 6464 static const struct hnae3_client_ops hns_roce_hw_v2_ops = { 6465 .init_instance = hns_roce_hw_v2_init_instance, 6466 .uninit_instance = hns_roce_hw_v2_uninit_instance, 6467 .reset_notify = hns_roce_hw_v2_reset_notify, 6468 }; 6469 6470 static struct hnae3_client hns_roce_hw_v2_client = { 6471 .name = "hns_roce_hw_v2", 6472 .type = HNAE3_CLIENT_ROCE, 6473 .ops = &hns_roce_hw_v2_ops, 6474 }; 6475 6476 static int __init hns_roce_hw_v2_init(void) 6477 { 6478 return hnae3_register_client(&hns_roce_hw_v2_client); 6479 } 6480 6481 static void __exit hns_roce_hw_v2_exit(void) 6482 { 6483 hnae3_unregister_client(&hns_roce_hw_v2_client); 6484 } 6485 6486 module_init(hns_roce_hw_v2_init); 6487 module_exit(hns_roce_hw_v2_exit); 6488 6489 MODULE_LICENSE("Dual BSD/GPL"); 6490 MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>"); 6491 MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>"); 6492 MODULE_AUTHOR("Shaobo Xu <xushaobo2@huawei.com>"); 6493 MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE Driver"); 6494