1 /* 2 * Copyright (c) 2016-2017 Hisilicon Limited. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/acpi.h> 34 #include <linux/etherdevice.h> 35 #include <linux/interrupt.h> 36 #include <linux/kernel.h> 37 #include <linux/types.h> 38 #include <net/addrconf.h> 39 #include <rdma/ib_addr.h> 40 #include <rdma/ib_cache.h> 41 #include <rdma/ib_umem.h> 42 #include <rdma/uverbs_ioctl.h> 43 44 #include "hnae3.h" 45 #include "hns_roce_common.h" 46 #include "hns_roce_device.h" 47 #include "hns_roce_cmd.h" 48 #include "hns_roce_hem.h" 49 #include "hns_roce_hw_v2.h" 50 51 static void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg, 52 struct ib_sge *sg) 53 { 54 dseg->lkey = cpu_to_le32(sg->lkey); 55 dseg->addr = cpu_to_le64(sg->addr); 56 dseg->len = cpu_to_le32(sg->length); 57 } 58 59 /* 60 * mapped-value = 1 + real-value 61 * The hns wr opcode real value is start from 0, In order to distinguish between 62 * initialized and uninitialized map values, we plus 1 to the actual value when 63 * defining the mapping, so that the validity can be identified by checking the 64 * mapped value is greater than 0. 65 */ 66 #define HR_OPC_MAP(ib_key, hr_key) \ 67 [IB_WR_ ## ib_key] = 1 + HNS_ROCE_V2_WQE_OP_ ## hr_key 68 69 static const u32 hns_roce_op_code[] = { 70 HR_OPC_MAP(RDMA_WRITE, RDMA_WRITE), 71 HR_OPC_MAP(RDMA_WRITE_WITH_IMM, RDMA_WRITE_WITH_IMM), 72 HR_OPC_MAP(SEND, SEND), 73 HR_OPC_MAP(SEND_WITH_IMM, SEND_WITH_IMM), 74 HR_OPC_MAP(RDMA_READ, RDMA_READ), 75 HR_OPC_MAP(ATOMIC_CMP_AND_SWP, ATOM_CMP_AND_SWAP), 76 HR_OPC_MAP(ATOMIC_FETCH_AND_ADD, ATOM_FETCH_AND_ADD), 77 HR_OPC_MAP(SEND_WITH_INV, SEND_WITH_INV), 78 HR_OPC_MAP(LOCAL_INV, LOCAL_INV), 79 HR_OPC_MAP(MASKED_ATOMIC_CMP_AND_SWP, ATOM_MSK_CMP_AND_SWAP), 80 HR_OPC_MAP(MASKED_ATOMIC_FETCH_AND_ADD, ATOM_MSK_FETCH_AND_ADD), 81 HR_OPC_MAP(REG_MR, FAST_REG_PMR), 82 }; 83 84 static u32 to_hr_opcode(u32 ib_opcode) 85 { 86 if (ib_opcode >= ARRAY_SIZE(hns_roce_op_code)) 87 return HNS_ROCE_V2_WQE_OP_MASK; 88 89 return hns_roce_op_code[ib_opcode] ? hns_roce_op_code[ib_opcode] - 1 : 90 HNS_ROCE_V2_WQE_OP_MASK; 91 } 92 93 static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, 94 void *wqe, const struct ib_reg_wr *wr) 95 { 96 struct hns_roce_mr *mr = to_hr_mr(wr->mr); 97 struct hns_roce_wqe_frmr_seg *fseg = wqe; 98 99 /* use ib_access_flags */ 100 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S, 101 wr->access & IB_ACCESS_MW_BIND ? 1 : 0); 102 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S, 103 wr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0); 104 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_RR_S, 105 wr->access & IB_ACCESS_REMOTE_READ ? 1 : 0); 106 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_RW_S, 107 wr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0); 108 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_LW_S, 109 wr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0); 110 111 /* Data structure reuse may lead to confusion */ 112 rc_sq_wqe->msg_len = cpu_to_le32(mr->pbl_ba & 0xffffffff); 113 rc_sq_wqe->inv_key = cpu_to_le32(mr->pbl_ba >> 32); 114 115 rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff); 116 rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32); 117 rc_sq_wqe->rkey = cpu_to_le32(wr->key); 118 rc_sq_wqe->va = cpu_to_le64(wr->mr->iova); 119 120 fseg->pbl_size = cpu_to_le32(mr->pbl_size); 121 roce_set_field(fseg->mode_buf_pg_sz, 122 V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M, 123 V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S, 124 mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET); 125 roce_set_bit(fseg->mode_buf_pg_sz, 126 V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0); 127 } 128 129 static void set_atomic_seg(const struct ib_send_wr *wr, void *wqe, 130 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, 131 int valid_num_sge) 132 { 133 struct hns_roce_wqe_atomic_seg *aseg; 134 135 set_data_seg_v2(wqe, wr->sg_list); 136 aseg = wqe + sizeof(struct hns_roce_v2_wqe_data_seg); 137 138 if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) { 139 aseg->fetchadd_swap_data = cpu_to_le64(atomic_wr(wr)->swap); 140 aseg->cmp_data = cpu_to_le64(atomic_wr(wr)->compare_add); 141 } else { 142 aseg->fetchadd_swap_data = 143 cpu_to_le64(atomic_wr(wr)->compare_add); 144 aseg->cmp_data = 0; 145 } 146 147 roce_set_field(rc_sq_wqe->byte_16, V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M, 148 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge); 149 } 150 151 static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr, 152 unsigned int *sge_ind, int valid_num_sge) 153 { 154 struct hns_roce_v2_wqe_data_seg *dseg; 155 struct ib_sge *sg; 156 int num_in_wqe = 0; 157 int extend_sge_num; 158 int fi_sge_num; 159 int se_sge_num; 160 int shift; 161 int i; 162 163 if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) 164 num_in_wqe = HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE; 165 extend_sge_num = valid_num_sge - num_in_wqe; 166 sg = wr->sg_list + num_in_wqe; 167 shift = qp->hr_buf.page_shift; 168 169 /* 170 * Check whether wr->num_sge sges are in the same page. If not, we 171 * should calculate how many sges in the first page and the second 172 * page. 173 */ 174 dseg = hns_roce_get_extend_sge(qp, (*sge_ind) & (qp->sge.sge_cnt - 1)); 175 fi_sge_num = (round_up((uintptr_t)dseg, 1 << shift) - 176 (uintptr_t)dseg) / 177 sizeof(struct hns_roce_v2_wqe_data_seg); 178 if (extend_sge_num > fi_sge_num) { 179 se_sge_num = extend_sge_num - fi_sge_num; 180 for (i = 0; i < fi_sge_num; i++) { 181 set_data_seg_v2(dseg++, sg + i); 182 (*sge_ind)++; 183 } 184 dseg = hns_roce_get_extend_sge(qp, 185 (*sge_ind) & (qp->sge.sge_cnt - 1)); 186 for (i = 0; i < se_sge_num; i++) { 187 set_data_seg_v2(dseg++, sg + fi_sge_num + i); 188 (*sge_ind)++; 189 } 190 } else { 191 for (i = 0; i < extend_sge_num; i++) { 192 set_data_seg_v2(dseg++, sg + i); 193 (*sge_ind)++; 194 } 195 } 196 } 197 198 static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr, 199 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, 200 void *wqe, unsigned int *sge_ind, 201 int valid_num_sge) 202 { 203 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 204 struct hns_roce_v2_wqe_data_seg *dseg = wqe; 205 struct ib_device *ibdev = &hr_dev->ib_dev; 206 struct hns_roce_qp *qp = to_hr_qp(ibqp); 207 int j = 0; 208 int i; 209 210 if (wr->send_flags & IB_SEND_INLINE && valid_num_sge) { 211 if (le32_to_cpu(rc_sq_wqe->msg_len) > 212 hr_dev->caps.max_sq_inline) { 213 ibdev_err(ibdev, "inline len(1-%d)=%d, illegal", 214 rc_sq_wqe->msg_len, 215 hr_dev->caps.max_sq_inline); 216 return -EINVAL; 217 } 218 219 if (wr->opcode == IB_WR_RDMA_READ) { 220 ibdev_err(ibdev, "Not support inline data!\n"); 221 return -EINVAL; 222 } 223 224 for (i = 0; i < wr->num_sge; i++) { 225 memcpy(wqe, ((void *)wr->sg_list[i].addr), 226 wr->sg_list[i].length); 227 wqe += wr->sg_list[i].length; 228 } 229 230 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S, 231 1); 232 } else { 233 if (valid_num_sge <= HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) { 234 for (i = 0; i < wr->num_sge; i++) { 235 if (likely(wr->sg_list[i].length)) { 236 set_data_seg_v2(dseg, wr->sg_list + i); 237 dseg++; 238 } 239 } 240 } else { 241 roce_set_field(rc_sq_wqe->byte_20, 242 V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M, 243 V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S, 244 (*sge_ind) & (qp->sge.sge_cnt - 1)); 245 246 for (i = 0; i < wr->num_sge && 247 j < HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE; i++) { 248 if (likely(wr->sg_list[i].length)) { 249 set_data_seg_v2(dseg, wr->sg_list + i); 250 dseg++; 251 j++; 252 } 253 } 254 255 set_extend_sge(qp, wr, sge_ind, valid_num_sge); 256 } 257 258 roce_set_field(rc_sq_wqe->byte_16, 259 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M, 260 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge); 261 } 262 263 return 0; 264 } 265 266 static int check_send_valid(struct hns_roce_dev *hr_dev, 267 struct hns_roce_qp *hr_qp) 268 { 269 struct ib_device *ibdev = &hr_dev->ib_dev; 270 struct ib_qp *ibqp = &hr_qp->ibqp; 271 272 if (unlikely(ibqp->qp_type != IB_QPT_RC && 273 ibqp->qp_type != IB_QPT_GSI && 274 ibqp->qp_type != IB_QPT_UD)) { 275 ibdev_err(ibdev, "Not supported QP(0x%x)type!\n", 276 ibqp->qp_type); 277 return -EOPNOTSUPP; 278 } else if (unlikely(hr_qp->state == IB_QPS_RESET || 279 hr_qp->state == IB_QPS_INIT || 280 hr_qp->state == IB_QPS_RTR)) { 281 ibdev_err(ibdev, "failed to post WQE, QP state %d!\n", 282 hr_qp->state); 283 return -EINVAL; 284 } else if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) { 285 ibdev_err(ibdev, "failed to post WQE, dev state %d!\n", 286 hr_dev->state); 287 return -EIO; 288 } 289 290 return 0; 291 } 292 293 static inline int calc_wr_sge_num(const struct ib_send_wr *wr, u32 *sge_len) 294 { 295 int valid_num = 0; 296 u32 len = 0; 297 int i; 298 299 for (i = 0; i < wr->num_sge; i++) { 300 if (likely(wr->sg_list[i].length)) { 301 len += wr->sg_list[i].length; 302 valid_num++; 303 } 304 } 305 306 *sge_len = len; 307 return valid_num; 308 } 309 310 static inline int set_ud_wqe(struct hns_roce_qp *qp, 311 const struct ib_send_wr *wr, 312 void *wqe, unsigned int *sge_idx, 313 unsigned int owner_bit) 314 { 315 struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device); 316 struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah); 317 struct hns_roce_v2_ud_send_wqe *ud_sq_wqe = wqe; 318 unsigned int curr_idx = *sge_idx; 319 int valid_num_sge; 320 u32 msg_len = 0; 321 bool loopback; 322 u8 *smac; 323 324 valid_num_sge = calc_wr_sge_num(wr, &msg_len); 325 memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe)); 326 327 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M, 328 V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]); 329 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M, 330 V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]); 331 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M, 332 V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]); 333 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M, 334 V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]); 335 roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_DMAC_4_M, 336 V2_UD_SEND_WQE_BYTE_48_DMAC_4_S, ah->av.mac[4]); 337 roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_DMAC_5_M, 338 V2_UD_SEND_WQE_BYTE_48_DMAC_5_S, ah->av.mac[5]); 339 340 /* MAC loopback */ 341 smac = (u8 *)hr_dev->dev_addr[qp->port]; 342 loopback = ether_addr_equal_unaligned(ah->av.mac, smac) ? 1 : 0; 343 344 roce_set_bit(ud_sq_wqe->byte_40, 345 V2_UD_SEND_WQE_BYTE_40_LBI_S, loopback); 346 347 roce_set_field(ud_sq_wqe->byte_4, 348 V2_UD_SEND_WQE_BYTE_4_OPCODE_M, 349 V2_UD_SEND_WQE_BYTE_4_OPCODE_S, 350 HNS_ROCE_V2_WQE_OP_SEND); 351 352 ud_sq_wqe->msg_len = cpu_to_le32(msg_len); 353 354 switch (wr->opcode) { 355 case IB_WR_SEND_WITH_IMM: 356 case IB_WR_RDMA_WRITE_WITH_IMM: 357 ud_sq_wqe->immtdata = cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); 358 break; 359 default: 360 ud_sq_wqe->immtdata = 0; 361 break; 362 } 363 364 /* Set sig attr */ 365 roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_CQE_S, 366 (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0); 367 368 /* Set se attr */ 369 roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_SE_S, 370 (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0); 371 372 roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_OWNER_S, 373 owner_bit); 374 375 roce_set_field(ud_sq_wqe->byte_16, V2_UD_SEND_WQE_BYTE_16_PD_M, 376 V2_UD_SEND_WQE_BYTE_16_PD_S, to_hr_pd(qp->ibqp.pd)->pdn); 377 378 roce_set_field(ud_sq_wqe->byte_16, V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M, 379 V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge); 380 381 roce_set_field(ud_sq_wqe->byte_20, 382 V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M, 383 V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S, 384 curr_idx & (qp->sge.sge_cnt - 1)); 385 386 roce_set_field(ud_sq_wqe->byte_24, V2_UD_SEND_WQE_BYTE_24_UDPSPN_M, 387 V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, 0); 388 ud_sq_wqe->qkey = cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ? 389 qp->qkey : ud_wr(wr)->remote_qkey); 390 roce_set_field(ud_sq_wqe->byte_32, V2_UD_SEND_WQE_BYTE_32_DQPN_M, 391 V2_UD_SEND_WQE_BYTE_32_DQPN_S, ud_wr(wr)->remote_qpn); 392 393 roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_VLAN_M, 394 V2_UD_SEND_WQE_BYTE_36_VLAN_S, ah->av.vlan_id); 395 roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M, 396 V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S, ah->av.hop_limit); 397 roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_TCLASS_M, 398 V2_UD_SEND_WQE_BYTE_36_TCLASS_S, ah->av.tclass); 399 roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M, 400 V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S, ah->av.flowlabel); 401 roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_SL_M, 402 V2_UD_SEND_WQE_BYTE_40_SL_S, ah->av.sl); 403 roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_PORTN_M, 404 V2_UD_SEND_WQE_BYTE_40_PORTN_S, qp->port); 405 406 roce_set_bit(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S, 407 ah->av.vlan_en ? 1 : 0); 408 roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M, 409 V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S, ah->av.gid_index); 410 411 memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN_V2); 412 413 set_extend_sge(qp, wr, &curr_idx, valid_num_sge); 414 415 *sge_idx = curr_idx; 416 417 return 0; 418 } 419 420 static inline int set_rc_wqe(struct hns_roce_qp *qp, 421 const struct ib_send_wr *wr, 422 void *wqe, unsigned int *sge_idx, 423 unsigned int owner_bit) 424 { 425 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe; 426 unsigned int curr_idx = *sge_idx; 427 int valid_num_sge; 428 u32 msg_len = 0; 429 int ret = 0; 430 431 valid_num_sge = calc_wr_sge_num(wr, &msg_len); 432 memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe)); 433 434 rc_sq_wqe->msg_len = cpu_to_le32(msg_len); 435 436 switch (wr->opcode) { 437 case IB_WR_SEND_WITH_IMM: 438 case IB_WR_RDMA_WRITE_WITH_IMM: 439 rc_sq_wqe->immtdata = cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); 440 break; 441 case IB_WR_SEND_WITH_INV: 442 rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey); 443 break; 444 default: 445 rc_sq_wqe->immtdata = 0; 446 break; 447 } 448 449 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_FENCE_S, 450 (wr->send_flags & IB_SEND_FENCE) ? 1 : 0); 451 452 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_SE_S, 453 (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0); 454 455 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_CQE_S, 456 (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0); 457 458 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OWNER_S, 459 owner_bit); 460 461 wqe += sizeof(struct hns_roce_v2_rc_send_wqe); 462 switch (wr->opcode) { 463 case IB_WR_RDMA_READ: 464 case IB_WR_RDMA_WRITE: 465 case IB_WR_RDMA_WRITE_WITH_IMM: 466 rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey); 467 rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr); 468 break; 469 case IB_WR_LOCAL_INV: 470 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_SO_S, 1); 471 rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey); 472 break; 473 case IB_WR_REG_MR: 474 set_frmr_seg(rc_sq_wqe, wqe, reg_wr(wr)); 475 break; 476 case IB_WR_ATOMIC_CMP_AND_SWP: 477 case IB_WR_ATOMIC_FETCH_AND_ADD: 478 rc_sq_wqe->rkey = cpu_to_le32(atomic_wr(wr)->rkey); 479 rc_sq_wqe->va = cpu_to_le64(atomic_wr(wr)->remote_addr); 480 break; 481 default: 482 break; 483 } 484 485 roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OPCODE_M, 486 V2_RC_SEND_WQE_BYTE_4_OPCODE_S, 487 to_hr_opcode(wr->opcode)); 488 489 if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || 490 wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) 491 set_atomic_seg(wr, wqe, rc_sq_wqe, valid_num_sge); 492 else if (wr->opcode != IB_WR_REG_MR) 493 ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe, 494 wqe, &curr_idx, valid_num_sge); 495 496 *sge_idx = curr_idx; 497 498 return ret; 499 } 500 501 static inline void update_sq_db(struct hns_roce_dev *hr_dev, 502 struct hns_roce_qp *qp) 503 { 504 /* 505 * Hip08 hardware cannot flush the WQEs in SQ if the QP state 506 * gets into errored mode. Hence, as a workaround to this 507 * hardware limitation, driver needs to assist in flushing. But 508 * the flushing operation uses mailbox to convey the QP state to 509 * the hardware and which can sleep due to the mutex protection 510 * around the mailbox calls. Hence, use the deferred flush for 511 * now. 512 */ 513 if (qp->state == IB_QPS_ERR) { 514 if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag)) 515 init_flush_work(hr_dev, qp); 516 } else { 517 struct hns_roce_v2_db sq_db = {}; 518 519 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M, 520 V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn); 521 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M, 522 V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB); 523 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M, 524 V2_DB_PARAMETER_IDX_S, 525 qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1)); 526 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M, 527 V2_DB_PARAMETER_SL_S, qp->sl); 528 529 hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg_l); 530 } 531 } 532 533 static int hns_roce_v2_post_send(struct ib_qp *ibqp, 534 const struct ib_send_wr *wr, 535 const struct ib_send_wr **bad_wr) 536 { 537 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 538 struct ib_device *ibdev = &hr_dev->ib_dev; 539 struct hns_roce_qp *qp = to_hr_qp(ibqp); 540 unsigned long flags = 0; 541 unsigned int owner_bit; 542 unsigned int sge_idx; 543 unsigned int wqe_idx; 544 void *wqe = NULL; 545 int nreq; 546 int ret; 547 548 spin_lock_irqsave(&qp->sq.lock, flags); 549 550 ret = check_send_valid(hr_dev, qp); 551 if (ret) { 552 *bad_wr = wr; 553 nreq = 0; 554 goto out; 555 } 556 557 sge_idx = qp->next_sge; 558 559 for (nreq = 0; wr; ++nreq, wr = wr->next) { 560 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { 561 ret = -ENOMEM; 562 *bad_wr = wr; 563 goto out; 564 } 565 566 wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1); 567 568 if (unlikely(wr->num_sge > qp->sq.max_gs)) { 569 ibdev_err(ibdev, "num_sge=%d > qp->sq.max_gs=%d\n", 570 wr->num_sge, qp->sq.max_gs); 571 ret = -EINVAL; 572 *bad_wr = wr; 573 goto out; 574 } 575 576 wqe = hns_roce_get_send_wqe(qp, wqe_idx); 577 qp->sq.wrid[wqe_idx] = wr->wr_id; 578 owner_bit = 579 ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1); 580 581 /* Corresponding to the QP type, wqe process separately */ 582 if (ibqp->qp_type == IB_QPT_GSI) 583 ret = set_ud_wqe(qp, wr, wqe, &sge_idx, owner_bit); 584 else if (ibqp->qp_type == IB_QPT_RC) 585 ret = set_rc_wqe(qp, wr, wqe, &sge_idx, owner_bit); 586 587 if (ret) { 588 *bad_wr = wr; 589 goto out; 590 } 591 } 592 593 out: 594 if (likely(nreq)) { 595 qp->sq.head += nreq; 596 qp->next_sge = sge_idx; 597 /* Memory barrier */ 598 wmb(); 599 update_sq_db(hr_dev, qp); 600 } 601 602 spin_unlock_irqrestore(&qp->sq.lock, flags); 603 604 return ret; 605 } 606 607 static int check_recv_valid(struct hns_roce_dev *hr_dev, 608 struct hns_roce_qp *hr_qp) 609 { 610 if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) 611 return -EIO; 612 else if (hr_qp->state == IB_QPS_RESET) 613 return -EINVAL; 614 615 return 0; 616 } 617 618 static int hns_roce_v2_post_recv(struct ib_qp *ibqp, 619 const struct ib_recv_wr *wr, 620 const struct ib_recv_wr **bad_wr) 621 { 622 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 623 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 624 struct ib_device *ibdev = &hr_dev->ib_dev; 625 struct hns_roce_v2_wqe_data_seg *dseg; 626 struct hns_roce_rinl_sge *sge_list; 627 unsigned long flags; 628 void *wqe = NULL; 629 u32 wqe_idx; 630 int nreq; 631 int ret; 632 int i; 633 634 spin_lock_irqsave(&hr_qp->rq.lock, flags); 635 636 ret = check_recv_valid(hr_dev, hr_qp); 637 if (ret) { 638 *bad_wr = wr; 639 nreq = 0; 640 goto out; 641 } 642 643 for (nreq = 0; wr; ++nreq, wr = wr->next) { 644 if (hns_roce_wq_overflow(&hr_qp->rq, nreq, 645 hr_qp->ibqp.recv_cq)) { 646 ret = -ENOMEM; 647 *bad_wr = wr; 648 goto out; 649 } 650 651 wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1); 652 653 if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) { 654 ibdev_err(ibdev, "rq:num_sge=%d >= qp->sq.max_gs=%d\n", 655 wr->num_sge, hr_qp->rq.max_gs); 656 ret = -EINVAL; 657 *bad_wr = wr; 658 goto out; 659 } 660 661 wqe = hns_roce_get_recv_wqe(hr_qp, wqe_idx); 662 dseg = (struct hns_roce_v2_wqe_data_seg *)wqe; 663 for (i = 0; i < wr->num_sge; i++) { 664 if (!wr->sg_list[i].length) 665 continue; 666 set_data_seg_v2(dseg, wr->sg_list + i); 667 dseg++; 668 } 669 670 if (i < hr_qp->rq.max_gs) { 671 dseg->lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY); 672 dseg->addr = 0; 673 } 674 675 /* rq support inline data */ 676 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) { 677 sge_list = hr_qp->rq_inl_buf.wqe_list[wqe_idx].sg_list; 678 hr_qp->rq_inl_buf.wqe_list[wqe_idx].sge_cnt = 679 (u32)wr->num_sge; 680 for (i = 0; i < wr->num_sge; i++) { 681 sge_list[i].addr = 682 (void *)(u64)wr->sg_list[i].addr; 683 sge_list[i].len = wr->sg_list[i].length; 684 } 685 } 686 687 hr_qp->rq.wrid[wqe_idx] = wr->wr_id; 688 } 689 690 out: 691 if (likely(nreq)) { 692 hr_qp->rq.head += nreq; 693 /* Memory barrier */ 694 wmb(); 695 696 /* 697 * Hip08 hardware cannot flush the WQEs in RQ if the QP state 698 * gets into errored mode. Hence, as a workaround to this 699 * hardware limitation, driver needs to assist in flushing. But 700 * the flushing operation uses mailbox to convey the QP state to 701 * the hardware and which can sleep due to the mutex protection 702 * around the mailbox calls. Hence, use the deferred flush for 703 * now. 704 */ 705 if (hr_qp->state == IB_QPS_ERR) { 706 if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, 707 &hr_qp->flush_flag)) 708 init_flush_work(hr_dev, hr_qp); 709 } else { 710 *hr_qp->rdb.db_record = hr_qp->rq.head & 0xffff; 711 } 712 } 713 spin_unlock_irqrestore(&hr_qp->rq.lock, flags); 714 715 return ret; 716 } 717 718 static int hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev, 719 unsigned long instance_stage, 720 unsigned long reset_stage) 721 { 722 /* When hardware reset has been completed once or more, we should stop 723 * sending mailbox&cmq&doorbell to hardware. If now in .init_instance() 724 * function, we should exit with error. If now at HNAE3_INIT_CLIENT 725 * stage of soft reset process, we should exit with error, and then 726 * HNAE3_INIT_CLIENT related process can rollback the operation like 727 * notifing hardware to free resources, HNAE3_INIT_CLIENT related 728 * process will exit with error to notify NIC driver to reschedule soft 729 * reset process once again. 730 */ 731 hr_dev->is_reset = true; 732 hr_dev->dis_db = true; 733 734 if (reset_stage == HNS_ROCE_STATE_RST_INIT || 735 instance_stage == HNS_ROCE_STATE_INIT) 736 return CMD_RST_PRC_EBUSY; 737 738 return CMD_RST_PRC_SUCCESS; 739 } 740 741 static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev, 742 unsigned long instance_stage, 743 unsigned long reset_stage) 744 { 745 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 746 struct hnae3_handle *handle = priv->handle; 747 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 748 749 /* When hardware reset is detected, we should stop sending mailbox&cmq& 750 * doorbell to hardware. If now in .init_instance() function, we should 751 * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset 752 * process, we should exit with error, and then HNAE3_INIT_CLIENT 753 * related process can rollback the operation like notifing hardware to 754 * free resources, HNAE3_INIT_CLIENT related process will exit with 755 * error to notify NIC driver to reschedule soft reset process once 756 * again. 757 */ 758 hr_dev->dis_db = true; 759 if (!ops->get_hw_reset_stat(handle)) 760 hr_dev->is_reset = true; 761 762 if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT || 763 instance_stage == HNS_ROCE_STATE_INIT) 764 return CMD_RST_PRC_EBUSY; 765 766 return CMD_RST_PRC_SUCCESS; 767 } 768 769 static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev) 770 { 771 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 772 struct hnae3_handle *handle = priv->handle; 773 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 774 775 /* When software reset is detected at .init_instance() function, we 776 * should stop sending mailbox&cmq&doorbell to hardware, and exit 777 * with error. 778 */ 779 hr_dev->dis_db = true; 780 if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt) 781 hr_dev->is_reset = true; 782 783 return CMD_RST_PRC_EBUSY; 784 } 785 786 static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev) 787 { 788 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 789 struct hnae3_handle *handle = priv->handle; 790 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 791 unsigned long instance_stage; /* the current instance stage */ 792 unsigned long reset_stage; /* the current reset stage */ 793 unsigned long reset_cnt; 794 bool sw_resetting; 795 bool hw_resetting; 796 797 if (hr_dev->is_reset) 798 return CMD_RST_PRC_SUCCESS; 799 800 /* Get information about reset from NIC driver or RoCE driver itself, 801 * the meaning of the following variables from NIC driver are described 802 * as below: 803 * reset_cnt -- The count value of completed hardware reset. 804 * hw_resetting -- Whether hardware device is resetting now. 805 * sw_resetting -- Whether NIC's software reset process is running now. 806 */ 807 instance_stage = handle->rinfo.instance_state; 808 reset_stage = handle->rinfo.reset_state; 809 reset_cnt = ops->ae_dev_reset_cnt(handle); 810 hw_resetting = ops->get_hw_reset_stat(handle); 811 sw_resetting = ops->ae_dev_resetting(handle); 812 813 if (reset_cnt != hr_dev->reset_cnt) 814 return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage, 815 reset_stage); 816 else if (hw_resetting) 817 return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage, 818 reset_stage); 819 else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT) 820 return hns_roce_v2_cmd_sw_resetting(hr_dev); 821 822 return 0; 823 } 824 825 static int hns_roce_cmq_space(struct hns_roce_v2_cmq_ring *ring) 826 { 827 int ntu = ring->next_to_use; 828 int ntc = ring->next_to_clean; 829 int used = (ntu - ntc + ring->desc_num) % ring->desc_num; 830 831 return ring->desc_num - used - 1; 832 } 833 834 static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev, 835 struct hns_roce_v2_cmq_ring *ring) 836 { 837 int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc); 838 839 ring->desc = kzalloc(size, GFP_KERNEL); 840 if (!ring->desc) 841 return -ENOMEM; 842 843 ring->desc_dma_addr = dma_map_single(hr_dev->dev, ring->desc, size, 844 DMA_BIDIRECTIONAL); 845 if (dma_mapping_error(hr_dev->dev, ring->desc_dma_addr)) { 846 ring->desc_dma_addr = 0; 847 kfree(ring->desc); 848 ring->desc = NULL; 849 return -ENOMEM; 850 } 851 852 return 0; 853 } 854 855 static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev, 856 struct hns_roce_v2_cmq_ring *ring) 857 { 858 dma_unmap_single(hr_dev->dev, ring->desc_dma_addr, 859 ring->desc_num * sizeof(struct hns_roce_cmq_desc), 860 DMA_BIDIRECTIONAL); 861 862 ring->desc_dma_addr = 0; 863 kfree(ring->desc); 864 } 865 866 static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type) 867 { 868 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 869 struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ? 870 &priv->cmq.csq : &priv->cmq.crq; 871 872 ring->flag = ring_type; 873 ring->next_to_clean = 0; 874 ring->next_to_use = 0; 875 876 return hns_roce_alloc_cmq_desc(hr_dev, ring); 877 } 878 879 static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type) 880 { 881 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 882 struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ? 883 &priv->cmq.csq : &priv->cmq.crq; 884 dma_addr_t dma = ring->desc_dma_addr; 885 886 if (ring_type == TYPE_CSQ) { 887 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, (u32)dma); 888 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG, 889 upper_32_bits(dma)); 890 roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG, 891 ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S); 892 roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0); 893 roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0); 894 } else { 895 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_L_REG, (u32)dma); 896 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG, 897 upper_32_bits(dma)); 898 roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG, 899 ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S); 900 roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0); 901 roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0); 902 } 903 } 904 905 static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev) 906 { 907 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 908 int ret; 909 910 /* Setup the queue entries for command queue */ 911 priv->cmq.csq.desc_num = CMD_CSQ_DESC_NUM; 912 priv->cmq.crq.desc_num = CMD_CRQ_DESC_NUM; 913 914 /* Setup the lock for command queue */ 915 spin_lock_init(&priv->cmq.csq.lock); 916 spin_lock_init(&priv->cmq.crq.lock); 917 918 /* Setup Tx write back timeout */ 919 priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT; 920 921 /* Init CSQ */ 922 ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CSQ); 923 if (ret) { 924 dev_err(hr_dev->dev, "Init CSQ error, ret = %d.\n", ret); 925 return ret; 926 } 927 928 /* Init CRQ */ 929 ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CRQ); 930 if (ret) { 931 dev_err(hr_dev->dev, "Init CRQ error, ret = %d.\n", ret); 932 goto err_crq; 933 } 934 935 /* Init CSQ REG */ 936 hns_roce_cmq_init_regs(hr_dev, TYPE_CSQ); 937 938 /* Init CRQ REG */ 939 hns_roce_cmq_init_regs(hr_dev, TYPE_CRQ); 940 941 return 0; 942 943 err_crq: 944 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq); 945 946 return ret; 947 } 948 949 static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev) 950 { 951 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 952 953 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq); 954 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq); 955 } 956 957 static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc, 958 enum hns_roce_opcode_type opcode, 959 bool is_read) 960 { 961 memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc)); 962 desc->opcode = cpu_to_le16(opcode); 963 desc->flag = 964 cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN); 965 if (is_read) 966 desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR); 967 else 968 desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR); 969 } 970 971 static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev) 972 { 973 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 974 u32 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG); 975 976 return head == priv->cmq.csq.next_to_use; 977 } 978 979 static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev) 980 { 981 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 982 struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq; 983 struct hns_roce_cmq_desc *desc; 984 u16 ntc = csq->next_to_clean; 985 u32 head; 986 int clean = 0; 987 988 desc = &csq->desc[ntc]; 989 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG); 990 while (head != ntc) { 991 memset(desc, 0, sizeof(*desc)); 992 ntc++; 993 if (ntc == csq->desc_num) 994 ntc = 0; 995 desc = &csq->desc[ntc]; 996 clean++; 997 } 998 csq->next_to_clean = ntc; 999 1000 return clean; 1001 } 1002 1003 static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev, 1004 struct hns_roce_cmq_desc *desc, int num) 1005 { 1006 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 1007 struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq; 1008 struct hns_roce_cmq_desc *desc_to_use; 1009 bool complete = false; 1010 u32 timeout = 0; 1011 int handle = 0; 1012 u16 desc_ret; 1013 int ret = 0; 1014 int ntc; 1015 1016 spin_lock_bh(&csq->lock); 1017 1018 if (num > hns_roce_cmq_space(csq)) { 1019 spin_unlock_bh(&csq->lock); 1020 return -EBUSY; 1021 } 1022 1023 /* 1024 * Record the location of desc in the cmq for this time 1025 * which will be use for hardware to write back 1026 */ 1027 ntc = csq->next_to_use; 1028 1029 while (handle < num) { 1030 desc_to_use = &csq->desc[csq->next_to_use]; 1031 *desc_to_use = desc[handle]; 1032 dev_dbg(hr_dev->dev, "set cmq desc:\n"); 1033 csq->next_to_use++; 1034 if (csq->next_to_use == csq->desc_num) 1035 csq->next_to_use = 0; 1036 handle++; 1037 } 1038 1039 /* Write to hardware */ 1040 roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, csq->next_to_use); 1041 1042 /* 1043 * If the command is sync, wait for the firmware to write back, 1044 * if multi descriptors to be sent, use the first one to check 1045 */ 1046 if (le16_to_cpu(desc->flag) & HNS_ROCE_CMD_FLAG_NO_INTR) { 1047 do { 1048 if (hns_roce_cmq_csq_done(hr_dev)) 1049 break; 1050 udelay(1); 1051 timeout++; 1052 } while (timeout < priv->cmq.tx_timeout); 1053 } 1054 1055 if (hns_roce_cmq_csq_done(hr_dev)) { 1056 complete = true; 1057 handle = 0; 1058 while (handle < num) { 1059 /* get the result of hardware write back */ 1060 desc_to_use = &csq->desc[ntc]; 1061 desc[handle] = *desc_to_use; 1062 dev_dbg(hr_dev->dev, "Get cmq desc:\n"); 1063 desc_ret = le16_to_cpu(desc[handle].retval); 1064 if (desc_ret == CMD_EXEC_SUCCESS) 1065 ret = 0; 1066 else 1067 ret = -EIO; 1068 priv->cmq.last_status = desc_ret; 1069 ntc++; 1070 handle++; 1071 if (ntc == csq->desc_num) 1072 ntc = 0; 1073 } 1074 } 1075 1076 if (!complete) 1077 ret = -EAGAIN; 1078 1079 /* clean the command send queue */ 1080 handle = hns_roce_cmq_csq_clean(hr_dev); 1081 if (handle != num) 1082 dev_warn(hr_dev->dev, "Cleaned %d, need to clean %d\n", 1083 handle, num); 1084 1085 spin_unlock_bh(&csq->lock); 1086 1087 return ret; 1088 } 1089 1090 static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev, 1091 struct hns_roce_cmq_desc *desc, int num) 1092 { 1093 int retval; 1094 int ret; 1095 1096 ret = hns_roce_v2_rst_process_cmd(hr_dev); 1097 if (ret == CMD_RST_PRC_SUCCESS) 1098 return 0; 1099 if (ret == CMD_RST_PRC_EBUSY) 1100 return -EBUSY; 1101 1102 ret = __hns_roce_cmq_send(hr_dev, desc, num); 1103 if (ret) { 1104 retval = hns_roce_v2_rst_process_cmd(hr_dev); 1105 if (retval == CMD_RST_PRC_SUCCESS) 1106 return 0; 1107 else if (retval == CMD_RST_PRC_EBUSY) 1108 return -EBUSY; 1109 } 1110 1111 return ret; 1112 } 1113 1114 static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev) 1115 { 1116 struct hns_roce_query_version *resp; 1117 struct hns_roce_cmq_desc desc; 1118 int ret; 1119 1120 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true); 1121 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1122 if (ret) 1123 return ret; 1124 1125 resp = (struct hns_roce_query_version *)desc.data; 1126 hr_dev->hw_rev = le16_to_cpu(resp->rocee_hw_version); 1127 hr_dev->vendor_id = hr_dev->pci_dev->vendor; 1128 1129 return 0; 1130 } 1131 1132 static bool hns_roce_func_clr_chk_rst(struct hns_roce_dev *hr_dev) 1133 { 1134 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 1135 struct hnae3_handle *handle = priv->handle; 1136 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 1137 unsigned long reset_cnt; 1138 bool sw_resetting; 1139 bool hw_resetting; 1140 1141 reset_cnt = ops->ae_dev_reset_cnt(handle); 1142 hw_resetting = ops->get_hw_reset_stat(handle); 1143 sw_resetting = ops->ae_dev_resetting(handle); 1144 1145 if (reset_cnt != hr_dev->reset_cnt || hw_resetting || sw_resetting) 1146 return true; 1147 1148 return false; 1149 } 1150 1151 static void hns_roce_func_clr_rst_prc(struct hns_roce_dev *hr_dev, int retval, 1152 int flag) 1153 { 1154 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 1155 struct hnae3_handle *handle = priv->handle; 1156 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 1157 unsigned long instance_stage; 1158 unsigned long reset_cnt; 1159 unsigned long end; 1160 bool sw_resetting; 1161 bool hw_resetting; 1162 1163 instance_stage = handle->rinfo.instance_state; 1164 reset_cnt = ops->ae_dev_reset_cnt(handle); 1165 hw_resetting = ops->get_hw_reset_stat(handle); 1166 sw_resetting = ops->ae_dev_resetting(handle); 1167 1168 if (reset_cnt != hr_dev->reset_cnt) { 1169 hr_dev->dis_db = true; 1170 hr_dev->is_reset = true; 1171 dev_info(hr_dev->dev, "Func clear success after reset.\n"); 1172 } else if (hw_resetting) { 1173 hr_dev->dis_db = true; 1174 1175 dev_warn(hr_dev->dev, 1176 "Func clear is pending, device in resetting state.\n"); 1177 end = HNS_ROCE_V2_HW_RST_TIMEOUT; 1178 while (end) { 1179 if (!ops->get_hw_reset_stat(handle)) { 1180 hr_dev->is_reset = true; 1181 dev_info(hr_dev->dev, 1182 "Func clear success after reset.\n"); 1183 return; 1184 } 1185 msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT); 1186 end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT; 1187 } 1188 1189 dev_warn(hr_dev->dev, "Func clear failed.\n"); 1190 } else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT) { 1191 hr_dev->dis_db = true; 1192 1193 dev_warn(hr_dev->dev, 1194 "Func clear is pending, device in resetting state.\n"); 1195 end = HNS_ROCE_V2_HW_RST_TIMEOUT; 1196 while (end) { 1197 if (ops->ae_dev_reset_cnt(handle) != 1198 hr_dev->reset_cnt) { 1199 hr_dev->is_reset = true; 1200 dev_info(hr_dev->dev, 1201 "Func clear success after sw reset\n"); 1202 return; 1203 } 1204 msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT); 1205 end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT; 1206 } 1207 1208 dev_warn(hr_dev->dev, "Func clear failed because of unfinished sw reset\n"); 1209 } else { 1210 if (retval && !flag) 1211 dev_warn(hr_dev->dev, 1212 "Func clear read failed, ret = %d.\n", retval); 1213 1214 dev_warn(hr_dev->dev, "Func clear failed.\n"); 1215 } 1216 } 1217 static void hns_roce_function_clear(struct hns_roce_dev *hr_dev) 1218 { 1219 bool fclr_write_fail_flag = false; 1220 struct hns_roce_func_clear *resp; 1221 struct hns_roce_cmq_desc desc; 1222 unsigned long end; 1223 int ret = 0; 1224 1225 if (hns_roce_func_clr_chk_rst(hr_dev)) 1226 goto out; 1227 1228 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false); 1229 resp = (struct hns_roce_func_clear *)desc.data; 1230 1231 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1232 if (ret) { 1233 fclr_write_fail_flag = true; 1234 dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n", 1235 ret); 1236 goto out; 1237 } 1238 1239 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL); 1240 end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS; 1241 while (end) { 1242 if (hns_roce_func_clr_chk_rst(hr_dev)) 1243 goto out; 1244 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT); 1245 end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT; 1246 1247 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, 1248 true); 1249 1250 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1251 if (ret) 1252 continue; 1253 1254 if (roce_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) { 1255 hr_dev->is_reset = true; 1256 return; 1257 } 1258 } 1259 1260 out: 1261 hns_roce_func_clr_rst_prc(hr_dev, ret, fclr_write_fail_flag); 1262 } 1263 1264 static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev) 1265 { 1266 struct hns_roce_query_fw_info *resp; 1267 struct hns_roce_cmq_desc desc; 1268 int ret; 1269 1270 hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true); 1271 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1272 if (ret) 1273 return ret; 1274 1275 resp = (struct hns_roce_query_fw_info *)desc.data; 1276 hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver)); 1277 1278 return 0; 1279 } 1280 1281 static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev) 1282 { 1283 struct hns_roce_cfg_global_param *req; 1284 struct hns_roce_cmq_desc desc; 1285 1286 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM, 1287 false); 1288 1289 req = (struct hns_roce_cfg_global_param *)desc.data; 1290 memset(req, 0, sizeof(*req)); 1291 roce_set_field(req->time_cfg_udp_port, 1292 CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M, 1293 CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S, 0x3e8); 1294 roce_set_field(req->time_cfg_udp_port, 1295 CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M, 1296 CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S, 0x12b7); 1297 1298 return hns_roce_cmq_send(hr_dev, &desc, 1); 1299 } 1300 1301 static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev) 1302 { 1303 struct hns_roce_cmq_desc desc[2]; 1304 struct hns_roce_pf_res_a *req_a; 1305 struct hns_roce_pf_res_b *req_b; 1306 int ret; 1307 int i; 1308 1309 for (i = 0; i < 2; i++) { 1310 hns_roce_cmq_setup_basic_desc(&desc[i], 1311 HNS_ROCE_OPC_QUERY_PF_RES, true); 1312 1313 if (i == 0) 1314 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1315 else 1316 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1317 } 1318 1319 ret = hns_roce_cmq_send(hr_dev, desc, 2); 1320 if (ret) 1321 return ret; 1322 1323 req_a = (struct hns_roce_pf_res_a *)desc[0].data; 1324 req_b = (struct hns_roce_pf_res_b *)desc[1].data; 1325 1326 hr_dev->caps.qpc_bt_num = roce_get_field(req_a->qpc_bt_idx_num, 1327 PF_RES_DATA_1_PF_QPC_BT_NUM_M, 1328 PF_RES_DATA_1_PF_QPC_BT_NUM_S); 1329 hr_dev->caps.srqc_bt_num = roce_get_field(req_a->srqc_bt_idx_num, 1330 PF_RES_DATA_2_PF_SRQC_BT_NUM_M, 1331 PF_RES_DATA_2_PF_SRQC_BT_NUM_S); 1332 hr_dev->caps.cqc_bt_num = roce_get_field(req_a->cqc_bt_idx_num, 1333 PF_RES_DATA_3_PF_CQC_BT_NUM_M, 1334 PF_RES_DATA_3_PF_CQC_BT_NUM_S); 1335 hr_dev->caps.mpt_bt_num = roce_get_field(req_a->mpt_bt_idx_num, 1336 PF_RES_DATA_4_PF_MPT_BT_NUM_M, 1337 PF_RES_DATA_4_PF_MPT_BT_NUM_S); 1338 1339 hr_dev->caps.sl_num = roce_get_field(req_b->qid_idx_sl_num, 1340 PF_RES_DATA_3_PF_SL_NUM_M, 1341 PF_RES_DATA_3_PF_SL_NUM_S); 1342 hr_dev->caps.sccc_bt_num = roce_get_field(req_b->sccc_bt_idx_num, 1343 PF_RES_DATA_4_PF_SCCC_BT_NUM_M, 1344 PF_RES_DATA_4_PF_SCCC_BT_NUM_S); 1345 1346 return 0; 1347 } 1348 1349 static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev) 1350 { 1351 struct hns_roce_pf_timer_res_a *req_a; 1352 struct hns_roce_cmq_desc desc[2]; 1353 int ret, i; 1354 1355 for (i = 0; i < 2; i++) { 1356 hns_roce_cmq_setup_basic_desc(&desc[i], 1357 HNS_ROCE_OPC_QUERY_PF_TIMER_RES, 1358 true); 1359 1360 if (i == 0) 1361 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1362 else 1363 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1364 } 1365 1366 ret = hns_roce_cmq_send(hr_dev, desc, 2); 1367 if (ret) 1368 return ret; 1369 1370 req_a = (struct hns_roce_pf_timer_res_a *)desc[0].data; 1371 1372 hr_dev->caps.qpc_timer_bt_num = 1373 roce_get_field(req_a->qpc_timer_bt_idx_num, 1374 PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M, 1375 PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S); 1376 hr_dev->caps.cqc_timer_bt_num = 1377 roce_get_field(req_a->cqc_timer_bt_idx_num, 1378 PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M, 1379 PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S); 1380 1381 return 0; 1382 } 1383 1384 static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, int vf_id) 1385 { 1386 struct hns_roce_cmq_desc desc; 1387 struct hns_roce_vf_switch *swt; 1388 int ret; 1389 1390 swt = (struct hns_roce_vf_switch *)desc.data; 1391 hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true); 1392 swt->rocee_sel |= cpu_to_le32(HNS_ICL_SWITCH_CMD_ROCEE_SEL); 1393 roce_set_field(swt->fun_id, VF_SWITCH_DATA_FUN_ID_VF_ID_M, 1394 VF_SWITCH_DATA_FUN_ID_VF_ID_S, vf_id); 1395 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1396 if (ret) 1397 return ret; 1398 1399 desc.flag = 1400 cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN); 1401 desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR); 1402 roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LPBK_S, 1); 1403 roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S, 0); 1404 roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S, 1); 1405 1406 return hns_roce_cmq_send(hr_dev, &desc, 1); 1407 } 1408 1409 static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev) 1410 { 1411 struct hns_roce_cmq_desc desc[2]; 1412 struct hns_roce_vf_res_a *req_a; 1413 struct hns_roce_vf_res_b *req_b; 1414 int i; 1415 1416 req_a = (struct hns_roce_vf_res_a *)desc[0].data; 1417 req_b = (struct hns_roce_vf_res_b *)desc[1].data; 1418 memset(req_a, 0, sizeof(*req_a)); 1419 memset(req_b, 0, sizeof(*req_b)); 1420 for (i = 0; i < 2; i++) { 1421 hns_roce_cmq_setup_basic_desc(&desc[i], 1422 HNS_ROCE_OPC_ALLOC_VF_RES, false); 1423 1424 if (i == 0) 1425 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1426 else 1427 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1428 } 1429 1430 roce_set_field(req_a->vf_qpc_bt_idx_num, 1431 VF_RES_A_DATA_1_VF_QPC_BT_IDX_M, 1432 VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0); 1433 roce_set_field(req_a->vf_qpc_bt_idx_num, 1434 VF_RES_A_DATA_1_VF_QPC_BT_NUM_M, 1435 VF_RES_A_DATA_1_VF_QPC_BT_NUM_S, HNS_ROCE_VF_QPC_BT_NUM); 1436 1437 roce_set_field(req_a->vf_srqc_bt_idx_num, 1438 VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M, 1439 VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0); 1440 roce_set_field(req_a->vf_srqc_bt_idx_num, 1441 VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M, 1442 VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S, 1443 HNS_ROCE_VF_SRQC_BT_NUM); 1444 1445 roce_set_field(req_a->vf_cqc_bt_idx_num, 1446 VF_RES_A_DATA_3_VF_CQC_BT_IDX_M, 1447 VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0); 1448 roce_set_field(req_a->vf_cqc_bt_idx_num, 1449 VF_RES_A_DATA_3_VF_CQC_BT_NUM_M, 1450 VF_RES_A_DATA_3_VF_CQC_BT_NUM_S, HNS_ROCE_VF_CQC_BT_NUM); 1451 1452 roce_set_field(req_a->vf_mpt_bt_idx_num, 1453 VF_RES_A_DATA_4_VF_MPT_BT_IDX_M, 1454 VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0); 1455 roce_set_field(req_a->vf_mpt_bt_idx_num, 1456 VF_RES_A_DATA_4_VF_MPT_BT_NUM_M, 1457 VF_RES_A_DATA_4_VF_MPT_BT_NUM_S, HNS_ROCE_VF_MPT_BT_NUM); 1458 1459 roce_set_field(req_a->vf_eqc_bt_idx_num, VF_RES_A_DATA_5_VF_EQC_IDX_M, 1460 VF_RES_A_DATA_5_VF_EQC_IDX_S, 0); 1461 roce_set_field(req_a->vf_eqc_bt_idx_num, VF_RES_A_DATA_5_VF_EQC_NUM_M, 1462 VF_RES_A_DATA_5_VF_EQC_NUM_S, HNS_ROCE_VF_EQC_NUM); 1463 1464 roce_set_field(req_b->vf_smac_idx_num, VF_RES_B_DATA_1_VF_SMAC_IDX_M, 1465 VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0); 1466 roce_set_field(req_b->vf_smac_idx_num, VF_RES_B_DATA_1_VF_SMAC_NUM_M, 1467 VF_RES_B_DATA_1_VF_SMAC_NUM_S, HNS_ROCE_VF_SMAC_NUM); 1468 1469 roce_set_field(req_b->vf_sgid_idx_num, VF_RES_B_DATA_2_VF_SGID_IDX_M, 1470 VF_RES_B_DATA_2_VF_SGID_IDX_S, 0); 1471 roce_set_field(req_b->vf_sgid_idx_num, VF_RES_B_DATA_2_VF_SGID_NUM_M, 1472 VF_RES_B_DATA_2_VF_SGID_NUM_S, HNS_ROCE_VF_SGID_NUM); 1473 1474 roce_set_field(req_b->vf_qid_idx_sl_num, VF_RES_B_DATA_3_VF_QID_IDX_M, 1475 VF_RES_B_DATA_3_VF_QID_IDX_S, 0); 1476 roce_set_field(req_b->vf_qid_idx_sl_num, VF_RES_B_DATA_3_VF_SL_NUM_M, 1477 VF_RES_B_DATA_3_VF_SL_NUM_S, HNS_ROCE_VF_SL_NUM); 1478 1479 roce_set_field(req_b->vf_sccc_idx_num, VF_RES_B_DATA_4_VF_SCCC_BT_IDX_M, 1480 VF_RES_B_DATA_4_VF_SCCC_BT_IDX_S, 0); 1481 roce_set_field(req_b->vf_sccc_idx_num, VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M, 1482 VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S, 1483 HNS_ROCE_VF_SCCC_BT_NUM); 1484 1485 return hns_roce_cmq_send(hr_dev, desc, 2); 1486 } 1487 1488 static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev) 1489 { 1490 u8 srqc_hop_num = hr_dev->caps.srqc_hop_num; 1491 u8 qpc_hop_num = hr_dev->caps.qpc_hop_num; 1492 u8 cqc_hop_num = hr_dev->caps.cqc_hop_num; 1493 u8 mpt_hop_num = hr_dev->caps.mpt_hop_num; 1494 u8 sccc_hop_num = hr_dev->caps.sccc_hop_num; 1495 struct hns_roce_cfg_bt_attr *req; 1496 struct hns_roce_cmq_desc desc; 1497 1498 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false); 1499 req = (struct hns_roce_cfg_bt_attr *)desc.data; 1500 memset(req, 0, sizeof(*req)); 1501 1502 roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M, 1503 CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S, 1504 hr_dev->caps.qpc_ba_pg_sz + PG_SHIFT_OFFSET); 1505 roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M, 1506 CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S, 1507 hr_dev->caps.qpc_buf_pg_sz + PG_SHIFT_OFFSET); 1508 roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M, 1509 CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S, 1510 qpc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : qpc_hop_num); 1511 1512 roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M, 1513 CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S, 1514 hr_dev->caps.srqc_ba_pg_sz + PG_SHIFT_OFFSET); 1515 roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M, 1516 CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S, 1517 hr_dev->caps.srqc_buf_pg_sz + PG_SHIFT_OFFSET); 1518 roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M, 1519 CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S, 1520 srqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : srqc_hop_num); 1521 1522 roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M, 1523 CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S, 1524 hr_dev->caps.cqc_ba_pg_sz + PG_SHIFT_OFFSET); 1525 roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M, 1526 CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S, 1527 hr_dev->caps.cqc_buf_pg_sz + PG_SHIFT_OFFSET); 1528 roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M, 1529 CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S, 1530 cqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : cqc_hop_num); 1531 1532 roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M, 1533 CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S, 1534 hr_dev->caps.mpt_ba_pg_sz + PG_SHIFT_OFFSET); 1535 roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M, 1536 CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S, 1537 hr_dev->caps.mpt_buf_pg_sz + PG_SHIFT_OFFSET); 1538 roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M, 1539 CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S, 1540 mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num); 1541 1542 roce_set_field(req->vf_sccc_cfg, 1543 CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_M, 1544 CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_S, 1545 hr_dev->caps.sccc_ba_pg_sz + PG_SHIFT_OFFSET); 1546 roce_set_field(req->vf_sccc_cfg, 1547 CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_M, 1548 CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_S, 1549 hr_dev->caps.sccc_buf_pg_sz + PG_SHIFT_OFFSET); 1550 roce_set_field(req->vf_sccc_cfg, 1551 CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_M, 1552 CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_S, 1553 sccc_hop_num == 1554 HNS_ROCE_HOP_NUM_0 ? 0 : sccc_hop_num); 1555 1556 return hns_roce_cmq_send(hr_dev, &desc, 1); 1557 } 1558 1559 static void set_default_caps(struct hns_roce_dev *hr_dev) 1560 { 1561 struct hns_roce_caps *caps = &hr_dev->caps; 1562 1563 caps->num_qps = HNS_ROCE_V2_MAX_QP_NUM; 1564 caps->max_wqes = HNS_ROCE_V2_MAX_WQE_NUM; 1565 caps->num_cqs = HNS_ROCE_V2_MAX_CQ_NUM; 1566 caps->num_srqs = HNS_ROCE_V2_MAX_SRQ_NUM; 1567 caps->min_cqes = HNS_ROCE_MIN_CQE_NUM; 1568 caps->max_cqes = HNS_ROCE_V2_MAX_CQE_NUM; 1569 caps->max_sq_sg = HNS_ROCE_V2_MAX_SQ_SGE_NUM; 1570 caps->max_extend_sg = HNS_ROCE_V2_MAX_EXTEND_SGE_NUM; 1571 caps->max_rq_sg = HNS_ROCE_V2_MAX_RQ_SGE_NUM; 1572 caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE; 1573 caps->num_uars = HNS_ROCE_V2_UAR_NUM; 1574 caps->phy_num_uars = HNS_ROCE_V2_PHY_UAR_NUM; 1575 caps->num_aeq_vectors = HNS_ROCE_V2_AEQE_VEC_NUM; 1576 caps->num_comp_vectors = HNS_ROCE_V2_COMP_VEC_NUM; 1577 caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM; 1578 caps->num_mtpts = HNS_ROCE_V2_MAX_MTPT_NUM; 1579 caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS; 1580 caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS; 1581 caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS; 1582 caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS; 1583 caps->num_pds = HNS_ROCE_V2_MAX_PD_NUM; 1584 caps->max_qp_init_rdma = HNS_ROCE_V2_MAX_QP_INIT_RDMA; 1585 caps->max_qp_dest_rdma = HNS_ROCE_V2_MAX_QP_DEST_RDMA; 1586 caps->max_sq_desc_sz = HNS_ROCE_V2_MAX_SQ_DESC_SZ; 1587 caps->max_rq_desc_sz = HNS_ROCE_V2_MAX_RQ_DESC_SZ; 1588 caps->max_srq_desc_sz = HNS_ROCE_V2_MAX_SRQ_DESC_SZ; 1589 caps->qpc_entry_sz = HNS_ROCE_V2_QPC_ENTRY_SZ; 1590 caps->irrl_entry_sz = HNS_ROCE_V2_IRRL_ENTRY_SZ; 1591 caps->trrl_entry_sz = HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ; 1592 caps->cqc_entry_sz = HNS_ROCE_V2_CQC_ENTRY_SZ; 1593 caps->srqc_entry_sz = HNS_ROCE_V2_SRQC_ENTRY_SZ; 1594 caps->mtpt_entry_sz = HNS_ROCE_V2_MTPT_ENTRY_SZ; 1595 caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ; 1596 caps->idx_entry_sz = HNS_ROCE_V2_IDX_ENTRY_SZ; 1597 caps->cq_entry_sz = HNS_ROCE_V2_CQE_ENTRY_SIZE; 1598 caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED; 1599 caps->reserved_lkey = 0; 1600 caps->reserved_pds = 0; 1601 caps->reserved_mrws = 1; 1602 caps->reserved_uars = 0; 1603 caps->reserved_cqs = 0; 1604 caps->reserved_srqs = 0; 1605 caps->reserved_qps = HNS_ROCE_V2_RSV_QPS; 1606 1607 caps->qpc_ba_pg_sz = 0; 1608 caps->qpc_buf_pg_sz = 0; 1609 caps->qpc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; 1610 caps->srqc_ba_pg_sz = 0; 1611 caps->srqc_buf_pg_sz = 0; 1612 caps->srqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; 1613 caps->cqc_ba_pg_sz = 0; 1614 caps->cqc_buf_pg_sz = 0; 1615 caps->cqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; 1616 caps->mpt_ba_pg_sz = 0; 1617 caps->mpt_buf_pg_sz = 0; 1618 caps->mpt_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; 1619 caps->mtt_ba_pg_sz = 0; 1620 caps->mtt_buf_pg_sz = 0; 1621 caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM; 1622 caps->wqe_sq_hop_num = HNS_ROCE_SQWQE_HOP_NUM; 1623 caps->wqe_sge_hop_num = HNS_ROCE_EXT_SGE_HOP_NUM; 1624 caps->wqe_rq_hop_num = HNS_ROCE_RQWQE_HOP_NUM; 1625 caps->cqe_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_256K; 1626 caps->cqe_buf_pg_sz = 0; 1627 caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM; 1628 caps->srqwqe_ba_pg_sz = 0; 1629 caps->srqwqe_buf_pg_sz = 0; 1630 caps->srqwqe_hop_num = HNS_ROCE_SRQWQE_HOP_NUM; 1631 caps->idx_ba_pg_sz = 0; 1632 caps->idx_buf_pg_sz = 0; 1633 caps->idx_hop_num = HNS_ROCE_IDX_HOP_NUM; 1634 caps->chunk_sz = HNS_ROCE_V2_TABLE_CHUNK_SIZE; 1635 1636 caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR | 1637 HNS_ROCE_CAP_FLAG_ROCE_V1_V2 | 1638 HNS_ROCE_CAP_FLAG_RQ_INLINE | 1639 HNS_ROCE_CAP_FLAG_RECORD_DB | 1640 HNS_ROCE_CAP_FLAG_SQ_RECORD_DB; 1641 1642 caps->pkey_table_len[0] = 1; 1643 caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM; 1644 caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM; 1645 caps->aeqe_depth = HNS_ROCE_V2_ASYNC_EQE_NUM; 1646 caps->local_ca_ack_delay = 0; 1647 caps->max_mtu = IB_MTU_4096; 1648 1649 caps->max_srq_wrs = HNS_ROCE_V2_MAX_SRQ_WR; 1650 caps->max_srq_sges = HNS_ROCE_V2_MAX_SRQ_SGE; 1651 1652 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B) { 1653 caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC | HNS_ROCE_CAP_FLAG_MW | 1654 HNS_ROCE_CAP_FLAG_SRQ | HNS_ROCE_CAP_FLAG_FRMR | 1655 HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL; 1656 1657 caps->num_qpc_timer = HNS_ROCE_V2_MAX_QPC_TIMER_NUM; 1658 caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ; 1659 caps->qpc_timer_ba_pg_sz = 0; 1660 caps->qpc_timer_buf_pg_sz = 0; 1661 caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0; 1662 caps->num_cqc_timer = HNS_ROCE_V2_MAX_CQC_TIMER_NUM; 1663 caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ; 1664 caps->cqc_timer_ba_pg_sz = 0; 1665 caps->cqc_timer_buf_pg_sz = 0; 1666 caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0; 1667 1668 caps->sccc_entry_sz = HNS_ROCE_V2_SCCC_ENTRY_SZ; 1669 caps->sccc_ba_pg_sz = 0; 1670 caps->sccc_buf_pg_sz = 0; 1671 caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM; 1672 } 1673 } 1674 1675 static void calc_pg_sz(int obj_num, int obj_size, int hop_num, int ctx_bt_num, 1676 int *buf_page_size, int *bt_page_size, u32 hem_type) 1677 { 1678 u64 obj_per_chunk; 1679 int bt_chunk_size = 1 << PAGE_SHIFT; 1680 int buf_chunk_size = 1 << PAGE_SHIFT; 1681 int obj_per_chunk_default = buf_chunk_size / obj_size; 1682 1683 *buf_page_size = 0; 1684 *bt_page_size = 0; 1685 1686 switch (hop_num) { 1687 case 3: 1688 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * 1689 (bt_chunk_size / BA_BYTE_LEN) * 1690 (bt_chunk_size / BA_BYTE_LEN) * 1691 obj_per_chunk_default; 1692 break; 1693 case 2: 1694 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * 1695 (bt_chunk_size / BA_BYTE_LEN) * 1696 obj_per_chunk_default; 1697 break; 1698 case 1: 1699 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * 1700 obj_per_chunk_default; 1701 break; 1702 case HNS_ROCE_HOP_NUM_0: 1703 obj_per_chunk = ctx_bt_num * obj_per_chunk_default; 1704 break; 1705 default: 1706 pr_err("Table %d not support hop_num = %d!\n", hem_type, 1707 hop_num); 1708 return; 1709 } 1710 1711 if (hem_type >= HEM_TYPE_MTT) 1712 *bt_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk)); 1713 else 1714 *buf_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk)); 1715 } 1716 1717 static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) 1718 { 1719 struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM]; 1720 struct hns_roce_caps *caps = &hr_dev->caps; 1721 struct hns_roce_query_pf_caps_a *resp_a; 1722 struct hns_roce_query_pf_caps_b *resp_b; 1723 struct hns_roce_query_pf_caps_c *resp_c; 1724 struct hns_roce_query_pf_caps_d *resp_d; 1725 struct hns_roce_query_pf_caps_e *resp_e; 1726 int ctx_hop_num; 1727 int pbl_hop_num; 1728 int ret; 1729 int i; 1730 1731 for (i = 0; i < HNS_ROCE_QUERY_PF_CAPS_CMD_NUM; i++) { 1732 hns_roce_cmq_setup_basic_desc(&desc[i], 1733 HNS_ROCE_OPC_QUERY_PF_CAPS_NUM, 1734 true); 1735 if (i < (HNS_ROCE_QUERY_PF_CAPS_CMD_NUM - 1)) 1736 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1737 else 1738 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1739 } 1740 1741 ret = hns_roce_cmq_send(hr_dev, desc, HNS_ROCE_QUERY_PF_CAPS_CMD_NUM); 1742 if (ret) 1743 return ret; 1744 1745 resp_a = (struct hns_roce_query_pf_caps_a *)desc[0].data; 1746 resp_b = (struct hns_roce_query_pf_caps_b *)desc[1].data; 1747 resp_c = (struct hns_roce_query_pf_caps_c *)desc[2].data; 1748 resp_d = (struct hns_roce_query_pf_caps_d *)desc[3].data; 1749 resp_e = (struct hns_roce_query_pf_caps_e *)desc[4].data; 1750 1751 caps->local_ca_ack_delay = resp_a->local_ca_ack_delay; 1752 caps->max_sq_sg = le16_to_cpu(resp_a->max_sq_sg); 1753 caps->max_sq_inline = le16_to_cpu(resp_a->max_sq_inline); 1754 caps->max_rq_sg = le16_to_cpu(resp_a->max_rq_sg); 1755 caps->max_extend_sg = le32_to_cpu(resp_a->max_extend_sg); 1756 caps->num_qpc_timer = le16_to_cpu(resp_a->num_qpc_timer); 1757 caps->num_cqc_timer = le16_to_cpu(resp_a->num_cqc_timer); 1758 caps->max_srq_sges = le16_to_cpu(resp_a->max_srq_sges); 1759 caps->num_aeq_vectors = resp_a->num_aeq_vectors; 1760 caps->num_other_vectors = resp_a->num_other_vectors; 1761 caps->max_sq_desc_sz = resp_a->max_sq_desc_sz; 1762 caps->max_rq_desc_sz = resp_a->max_rq_desc_sz; 1763 caps->max_srq_desc_sz = resp_a->max_srq_desc_sz; 1764 caps->cq_entry_sz = resp_a->cq_entry_sz; 1765 1766 caps->mtpt_entry_sz = resp_b->mtpt_entry_sz; 1767 caps->irrl_entry_sz = resp_b->irrl_entry_sz; 1768 caps->trrl_entry_sz = resp_b->trrl_entry_sz; 1769 caps->cqc_entry_sz = resp_b->cqc_entry_sz; 1770 caps->srqc_entry_sz = resp_b->srqc_entry_sz; 1771 caps->idx_entry_sz = resp_b->idx_entry_sz; 1772 caps->sccc_entry_sz = resp_b->scc_ctx_entry_sz; 1773 caps->max_mtu = resp_b->max_mtu; 1774 caps->qpc_entry_sz = le16_to_cpu(resp_b->qpc_entry_sz); 1775 caps->min_cqes = resp_b->min_cqes; 1776 caps->min_wqes = resp_b->min_wqes; 1777 caps->page_size_cap = le32_to_cpu(resp_b->page_size_cap); 1778 caps->pkey_table_len[0] = resp_b->pkey_table_len; 1779 caps->phy_num_uars = resp_b->phy_num_uars; 1780 ctx_hop_num = resp_b->ctx_hop_num; 1781 pbl_hop_num = resp_b->pbl_hop_num; 1782 1783 caps->num_pds = 1 << roce_get_field(resp_c->cap_flags_num_pds, 1784 V2_QUERY_PF_CAPS_C_NUM_PDS_M, 1785 V2_QUERY_PF_CAPS_C_NUM_PDS_S); 1786 caps->flags = roce_get_field(resp_c->cap_flags_num_pds, 1787 V2_QUERY_PF_CAPS_C_CAP_FLAGS_M, 1788 V2_QUERY_PF_CAPS_C_CAP_FLAGS_S); 1789 caps->num_cqs = 1 << roce_get_field(resp_c->max_gid_num_cqs, 1790 V2_QUERY_PF_CAPS_C_NUM_CQS_M, 1791 V2_QUERY_PF_CAPS_C_NUM_CQS_S); 1792 caps->gid_table_len[0] = roce_get_field(resp_c->max_gid_num_cqs, 1793 V2_QUERY_PF_CAPS_C_MAX_GID_M, 1794 V2_QUERY_PF_CAPS_C_MAX_GID_S); 1795 caps->max_cqes = 1 << roce_get_field(resp_c->cq_depth, 1796 V2_QUERY_PF_CAPS_C_CQ_DEPTH_M, 1797 V2_QUERY_PF_CAPS_C_CQ_DEPTH_S); 1798 caps->num_mtpts = 1 << roce_get_field(resp_c->num_mrws, 1799 V2_QUERY_PF_CAPS_C_NUM_MRWS_M, 1800 V2_QUERY_PF_CAPS_C_NUM_MRWS_S); 1801 caps->num_qps = 1 << roce_get_field(resp_c->ord_num_qps, 1802 V2_QUERY_PF_CAPS_C_NUM_QPS_M, 1803 V2_QUERY_PF_CAPS_C_NUM_QPS_S); 1804 caps->max_qp_init_rdma = roce_get_field(resp_c->ord_num_qps, 1805 V2_QUERY_PF_CAPS_C_MAX_ORD_M, 1806 V2_QUERY_PF_CAPS_C_MAX_ORD_S); 1807 caps->max_qp_dest_rdma = caps->max_qp_init_rdma; 1808 caps->max_wqes = 1 << le16_to_cpu(resp_c->sq_depth); 1809 caps->num_srqs = 1 << roce_get_field(resp_d->wq_hop_num_max_srqs, 1810 V2_QUERY_PF_CAPS_D_NUM_SRQS_M, 1811 V2_QUERY_PF_CAPS_D_NUM_SRQS_S); 1812 caps->max_srq_wrs = 1 << le16_to_cpu(resp_d->srq_depth); 1813 caps->ceqe_depth = 1 << roce_get_field(resp_d->num_ceqs_ceq_depth, 1814 V2_QUERY_PF_CAPS_D_CEQ_DEPTH_M, 1815 V2_QUERY_PF_CAPS_D_CEQ_DEPTH_S); 1816 caps->num_comp_vectors = roce_get_field(resp_d->num_ceqs_ceq_depth, 1817 V2_QUERY_PF_CAPS_D_NUM_CEQS_M, 1818 V2_QUERY_PF_CAPS_D_NUM_CEQS_S); 1819 caps->aeqe_depth = 1 << roce_get_field(resp_d->arm_st_aeq_depth, 1820 V2_QUERY_PF_CAPS_D_AEQ_DEPTH_M, 1821 V2_QUERY_PF_CAPS_D_AEQ_DEPTH_S); 1822 caps->default_aeq_arm_st = roce_get_field(resp_d->arm_st_aeq_depth, 1823 V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_M, 1824 V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_S); 1825 caps->default_ceq_arm_st = roce_get_field(resp_d->arm_st_aeq_depth, 1826 V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_M, 1827 V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_S); 1828 caps->reserved_pds = roce_get_field(resp_d->num_uars_rsv_pds, 1829 V2_QUERY_PF_CAPS_D_RSV_PDS_M, 1830 V2_QUERY_PF_CAPS_D_RSV_PDS_S); 1831 caps->num_uars = 1 << roce_get_field(resp_d->num_uars_rsv_pds, 1832 V2_QUERY_PF_CAPS_D_NUM_UARS_M, 1833 V2_QUERY_PF_CAPS_D_NUM_UARS_S); 1834 caps->reserved_qps = roce_get_field(resp_d->rsv_uars_rsv_qps, 1835 V2_QUERY_PF_CAPS_D_RSV_QPS_M, 1836 V2_QUERY_PF_CAPS_D_RSV_QPS_S); 1837 caps->reserved_uars = roce_get_field(resp_d->rsv_uars_rsv_qps, 1838 V2_QUERY_PF_CAPS_D_RSV_UARS_M, 1839 V2_QUERY_PF_CAPS_D_RSV_UARS_S); 1840 caps->reserved_mrws = roce_get_field(resp_e->chunk_size_shift_rsv_mrws, 1841 V2_QUERY_PF_CAPS_E_RSV_MRWS_M, 1842 V2_QUERY_PF_CAPS_E_RSV_MRWS_S); 1843 caps->chunk_sz = 1 << roce_get_field(resp_e->chunk_size_shift_rsv_mrws, 1844 V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_M, 1845 V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_S); 1846 caps->reserved_cqs = roce_get_field(resp_e->rsv_cqs, 1847 V2_QUERY_PF_CAPS_E_RSV_CQS_M, 1848 V2_QUERY_PF_CAPS_E_RSV_CQS_S); 1849 caps->reserved_srqs = roce_get_field(resp_e->rsv_srqs, 1850 V2_QUERY_PF_CAPS_E_RSV_SRQS_M, 1851 V2_QUERY_PF_CAPS_E_RSV_SRQS_S); 1852 caps->reserved_lkey = roce_get_field(resp_e->rsv_lkey, 1853 V2_QUERY_PF_CAPS_E_RSV_LKEYS_M, 1854 V2_QUERY_PF_CAPS_E_RSV_LKEYS_S); 1855 caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt); 1856 caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period); 1857 caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt); 1858 caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period); 1859 1860 caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ; 1861 caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ; 1862 caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ; 1863 caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS; 1864 caps->mtt_ba_pg_sz = 0; 1865 caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS; 1866 caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS; 1867 caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS; 1868 1869 caps->qpc_hop_num = ctx_hop_num; 1870 caps->srqc_hop_num = ctx_hop_num; 1871 caps->cqc_hop_num = ctx_hop_num; 1872 caps->mpt_hop_num = ctx_hop_num; 1873 caps->mtt_hop_num = pbl_hop_num; 1874 caps->cqe_hop_num = pbl_hop_num; 1875 caps->srqwqe_hop_num = pbl_hop_num; 1876 caps->idx_hop_num = pbl_hop_num; 1877 caps->wqe_sq_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs, 1878 V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_M, 1879 V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_S); 1880 caps->wqe_sge_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs, 1881 V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_M, 1882 V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_S); 1883 caps->wqe_rq_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs, 1884 V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_M, 1885 V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_S); 1886 1887 calc_pg_sz(caps->num_qps, caps->qpc_entry_sz, caps->qpc_hop_num, 1888 caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz, 1889 HEM_TYPE_QPC); 1890 calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num, 1891 caps->mpt_bt_num, &caps->mpt_buf_pg_sz, &caps->mpt_ba_pg_sz, 1892 HEM_TYPE_MTPT); 1893 calc_pg_sz(caps->num_cqs, caps->cqc_entry_sz, caps->cqc_hop_num, 1894 caps->cqc_bt_num, &caps->cqc_buf_pg_sz, &caps->cqc_ba_pg_sz, 1895 HEM_TYPE_CQC); 1896 calc_pg_sz(caps->num_srqs, caps->srqc_entry_sz, caps->srqc_hop_num, 1897 caps->srqc_bt_num, &caps->srqc_buf_pg_sz, 1898 &caps->srqc_ba_pg_sz, HEM_TYPE_SRQC); 1899 1900 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B) { 1901 caps->sccc_hop_num = ctx_hop_num; 1902 caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0; 1903 caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0; 1904 1905 calc_pg_sz(caps->num_qps, caps->sccc_entry_sz, 1906 caps->sccc_hop_num, caps->sccc_bt_num, 1907 &caps->sccc_buf_pg_sz, &caps->sccc_ba_pg_sz, 1908 HEM_TYPE_SCCC); 1909 calc_pg_sz(caps->num_cqc_timer, caps->cqc_timer_entry_sz, 1910 caps->cqc_timer_hop_num, caps->cqc_timer_bt_num, 1911 &caps->cqc_timer_buf_pg_sz, 1912 &caps->cqc_timer_ba_pg_sz, HEM_TYPE_CQC_TIMER); 1913 } 1914 1915 calc_pg_sz(caps->num_cqe_segs, caps->mtt_entry_sz, caps->cqe_hop_num, 1916 1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE); 1917 calc_pg_sz(caps->num_srqwqe_segs, caps->mtt_entry_sz, 1918 caps->srqwqe_hop_num, 1, &caps->srqwqe_buf_pg_sz, 1919 &caps->srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE); 1920 calc_pg_sz(caps->num_idx_segs, caps->idx_entry_sz, caps->idx_hop_num, 1921 1, &caps->idx_buf_pg_sz, &caps->idx_ba_pg_sz, HEM_TYPE_IDX); 1922 1923 return 0; 1924 } 1925 1926 static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) 1927 { 1928 struct hns_roce_caps *caps = &hr_dev->caps; 1929 int ret; 1930 1931 ret = hns_roce_cmq_query_hw_info(hr_dev); 1932 if (ret) { 1933 dev_err(hr_dev->dev, "Query hardware version fail, ret = %d.\n", 1934 ret); 1935 return ret; 1936 } 1937 1938 ret = hns_roce_query_fw_ver(hr_dev); 1939 if (ret) { 1940 dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n", 1941 ret); 1942 return ret; 1943 } 1944 1945 ret = hns_roce_config_global_param(hr_dev); 1946 if (ret) { 1947 dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n", 1948 ret); 1949 return ret; 1950 } 1951 1952 /* Get pf resource owned by every pf */ 1953 ret = hns_roce_query_pf_resource(hr_dev); 1954 if (ret) { 1955 dev_err(hr_dev->dev, "Query pf resource fail, ret = %d.\n", 1956 ret); 1957 return ret; 1958 } 1959 1960 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B) { 1961 ret = hns_roce_query_pf_timer_resource(hr_dev); 1962 if (ret) { 1963 dev_err(hr_dev->dev, 1964 "Query pf timer resource fail, ret = %d.\n", 1965 ret); 1966 return ret; 1967 } 1968 1969 ret = hns_roce_set_vf_switch_param(hr_dev, 0); 1970 if (ret) { 1971 dev_err(hr_dev->dev, 1972 "Set function switch param fail, ret = %d.\n", 1973 ret); 1974 return ret; 1975 } 1976 } 1977 1978 hr_dev->vendor_part_id = hr_dev->pci_dev->device; 1979 hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid); 1980 1981 caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS; 1982 caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS; 1983 caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS; 1984 caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS; 1985 1986 caps->pbl_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_16K; 1987 caps->pbl_buf_pg_sz = 0; 1988 caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM; 1989 caps->eqe_ba_pg_sz = 0; 1990 caps->eqe_buf_pg_sz = 0; 1991 caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM; 1992 caps->tsq_buf_pg_sz = 0; 1993 1994 ret = hns_roce_query_pf_caps(hr_dev); 1995 if (ret) 1996 set_default_caps(hr_dev); 1997 1998 ret = hns_roce_alloc_vf_resource(hr_dev); 1999 if (ret) { 2000 dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n", 2001 ret); 2002 return ret; 2003 } 2004 2005 ret = hns_roce_v2_set_bt(hr_dev); 2006 if (ret) 2007 dev_err(hr_dev->dev, "Configure bt attribute fail, ret = %d.\n", 2008 ret); 2009 2010 return ret; 2011 } 2012 2013 static int hns_roce_config_link_table(struct hns_roce_dev *hr_dev, 2014 enum hns_roce_link_table_type type) 2015 { 2016 struct hns_roce_cmq_desc desc[2]; 2017 struct hns_roce_cfg_llm_a *req_a = 2018 (struct hns_roce_cfg_llm_a *)desc[0].data; 2019 struct hns_roce_cfg_llm_b *req_b = 2020 (struct hns_roce_cfg_llm_b *)desc[1].data; 2021 struct hns_roce_v2_priv *priv = hr_dev->priv; 2022 struct hns_roce_link_table *link_tbl; 2023 struct hns_roce_link_table_entry *entry; 2024 enum hns_roce_opcode_type opcode; 2025 u32 page_num; 2026 int i; 2027 2028 switch (type) { 2029 case TSQ_LINK_TABLE: 2030 link_tbl = &priv->tsq; 2031 opcode = HNS_ROCE_OPC_CFG_EXT_LLM; 2032 break; 2033 case TPQ_LINK_TABLE: 2034 link_tbl = &priv->tpq; 2035 opcode = HNS_ROCE_OPC_CFG_TMOUT_LLM; 2036 break; 2037 default: 2038 return -EINVAL; 2039 } 2040 2041 page_num = link_tbl->npages; 2042 entry = link_tbl->table.buf; 2043 memset(req_a, 0, sizeof(*req_a)); 2044 memset(req_b, 0, sizeof(*req_b)); 2045 2046 for (i = 0; i < 2; i++) { 2047 hns_roce_cmq_setup_basic_desc(&desc[i], opcode, false); 2048 2049 if (i == 0) 2050 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 2051 else 2052 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 2053 2054 if (i == 0) { 2055 req_a->base_addr_l = 2056 cpu_to_le32(link_tbl->table.map & 0xffffffff); 2057 req_a->base_addr_h = 2058 cpu_to_le32(link_tbl->table.map >> 32); 2059 roce_set_field(req_a->depth_pgsz_init_en, 2060 CFG_LLM_QUE_DEPTH_M, CFG_LLM_QUE_DEPTH_S, 2061 link_tbl->npages); 2062 roce_set_field(req_a->depth_pgsz_init_en, 2063 CFG_LLM_QUE_PGSZ_M, CFG_LLM_QUE_PGSZ_S, 2064 link_tbl->pg_sz); 2065 req_a->head_ba_l = cpu_to_le32(entry[0].blk_ba0); 2066 req_a->head_ba_h_nxtptr = 2067 cpu_to_le32(entry[0].blk_ba1_nxt_ptr); 2068 roce_set_field(req_a->head_ptr, CFG_LLM_HEAD_PTR_M, 2069 CFG_LLM_HEAD_PTR_S, 0); 2070 } else { 2071 req_b->tail_ba_l = 2072 cpu_to_le32(entry[page_num - 1].blk_ba0); 2073 roce_set_field(req_b->tail_ba_h, CFG_LLM_TAIL_BA_H_M, 2074 CFG_LLM_TAIL_BA_H_S, 2075 entry[page_num - 1].blk_ba1_nxt_ptr & 2076 HNS_ROCE_LINK_TABLE_BA1_M); 2077 roce_set_field(req_b->tail_ptr, CFG_LLM_TAIL_PTR_M, 2078 CFG_LLM_TAIL_PTR_S, 2079 (entry[page_num - 2].blk_ba1_nxt_ptr & 2080 HNS_ROCE_LINK_TABLE_NXT_PTR_M) >> 2081 HNS_ROCE_LINK_TABLE_NXT_PTR_S); 2082 } 2083 } 2084 roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_INIT_EN_M, 2085 CFG_LLM_INIT_EN_S, 1); 2086 2087 return hns_roce_cmq_send(hr_dev, desc, 2); 2088 } 2089 2090 static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev, 2091 enum hns_roce_link_table_type type) 2092 { 2093 struct hns_roce_v2_priv *priv = hr_dev->priv; 2094 struct hns_roce_link_table *link_tbl; 2095 struct hns_roce_link_table_entry *entry; 2096 struct device *dev = hr_dev->dev; 2097 u32 buf_chk_sz; 2098 dma_addr_t t; 2099 int func_num = 1; 2100 int pg_num_a; 2101 int pg_num_b; 2102 int pg_num; 2103 int size; 2104 int i; 2105 2106 switch (type) { 2107 case TSQ_LINK_TABLE: 2108 link_tbl = &priv->tsq; 2109 buf_chk_sz = 1 << (hr_dev->caps.tsq_buf_pg_sz + PAGE_SHIFT); 2110 pg_num_a = hr_dev->caps.num_qps * 8 / buf_chk_sz; 2111 pg_num_b = hr_dev->caps.sl_num * 4 + 2; 2112 break; 2113 case TPQ_LINK_TABLE: 2114 link_tbl = &priv->tpq; 2115 buf_chk_sz = 1 << (hr_dev->caps.tpq_buf_pg_sz + PAGE_SHIFT); 2116 pg_num_a = hr_dev->caps.num_cqs * 4 / buf_chk_sz; 2117 pg_num_b = 2 * 4 * func_num + 2; 2118 break; 2119 default: 2120 return -EINVAL; 2121 } 2122 2123 pg_num = max(pg_num_a, pg_num_b); 2124 size = pg_num * sizeof(struct hns_roce_link_table_entry); 2125 2126 link_tbl->table.buf = dma_alloc_coherent(dev, size, 2127 &link_tbl->table.map, 2128 GFP_KERNEL); 2129 if (!link_tbl->table.buf) 2130 goto out; 2131 2132 link_tbl->pg_list = kcalloc(pg_num, sizeof(*link_tbl->pg_list), 2133 GFP_KERNEL); 2134 if (!link_tbl->pg_list) 2135 goto err_kcalloc_failed; 2136 2137 entry = link_tbl->table.buf; 2138 for (i = 0; i < pg_num; ++i) { 2139 link_tbl->pg_list[i].buf = dma_alloc_coherent(dev, buf_chk_sz, 2140 &t, GFP_KERNEL); 2141 if (!link_tbl->pg_list[i].buf) 2142 goto err_alloc_buf_failed; 2143 2144 link_tbl->pg_list[i].map = t; 2145 2146 entry[i].blk_ba0 = (u32)(t >> 12); 2147 entry[i].blk_ba1_nxt_ptr = (u32)(t >> 44); 2148 2149 if (i < (pg_num - 1)) 2150 entry[i].blk_ba1_nxt_ptr |= 2151 (i + 1) << HNS_ROCE_LINK_TABLE_NXT_PTR_S; 2152 2153 } 2154 link_tbl->npages = pg_num; 2155 link_tbl->pg_sz = buf_chk_sz; 2156 2157 return hns_roce_config_link_table(hr_dev, type); 2158 2159 err_alloc_buf_failed: 2160 for (i -= 1; i >= 0; i--) 2161 dma_free_coherent(dev, buf_chk_sz, 2162 link_tbl->pg_list[i].buf, 2163 link_tbl->pg_list[i].map); 2164 kfree(link_tbl->pg_list); 2165 2166 err_kcalloc_failed: 2167 dma_free_coherent(dev, size, link_tbl->table.buf, 2168 link_tbl->table.map); 2169 2170 out: 2171 return -ENOMEM; 2172 } 2173 2174 static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev, 2175 struct hns_roce_link_table *link_tbl) 2176 { 2177 struct device *dev = hr_dev->dev; 2178 int size; 2179 int i; 2180 2181 size = link_tbl->npages * sizeof(struct hns_roce_link_table_entry); 2182 2183 for (i = 0; i < link_tbl->npages; ++i) 2184 if (link_tbl->pg_list[i].buf) 2185 dma_free_coherent(dev, link_tbl->pg_sz, 2186 link_tbl->pg_list[i].buf, 2187 link_tbl->pg_list[i].map); 2188 kfree(link_tbl->pg_list); 2189 2190 dma_free_coherent(dev, size, link_tbl->table.buf, 2191 link_tbl->table.map); 2192 } 2193 2194 static int hns_roce_v2_init(struct hns_roce_dev *hr_dev) 2195 { 2196 struct hns_roce_v2_priv *priv = hr_dev->priv; 2197 int qpc_count, cqc_count; 2198 int ret, i; 2199 2200 /* TSQ includes SQ doorbell and ack doorbell */ 2201 ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE); 2202 if (ret) { 2203 dev_err(hr_dev->dev, "TSQ init failed, ret = %d.\n", ret); 2204 return ret; 2205 } 2206 2207 ret = hns_roce_init_link_table(hr_dev, TPQ_LINK_TABLE); 2208 if (ret) { 2209 dev_err(hr_dev->dev, "TPQ init failed, ret = %d.\n", ret); 2210 goto err_tpq_init_failed; 2211 } 2212 2213 /* Alloc memory for QPC Timer buffer space chunk */ 2214 for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num; 2215 qpc_count++) { 2216 ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table, 2217 qpc_count); 2218 if (ret) { 2219 dev_err(hr_dev->dev, "QPC Timer get failed\n"); 2220 goto err_qpc_timer_failed; 2221 } 2222 } 2223 2224 /* Alloc memory for CQC Timer buffer space chunk */ 2225 for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num; 2226 cqc_count++) { 2227 ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table, 2228 cqc_count); 2229 if (ret) { 2230 dev_err(hr_dev->dev, "CQC Timer get failed\n"); 2231 goto err_cqc_timer_failed; 2232 } 2233 } 2234 2235 return 0; 2236 2237 err_cqc_timer_failed: 2238 for (i = 0; i < cqc_count; i++) 2239 hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i); 2240 2241 err_qpc_timer_failed: 2242 for (i = 0; i < qpc_count; i++) 2243 hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i); 2244 2245 hns_roce_free_link_table(hr_dev, &priv->tpq); 2246 2247 err_tpq_init_failed: 2248 hns_roce_free_link_table(hr_dev, &priv->tsq); 2249 2250 return ret; 2251 } 2252 2253 static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev) 2254 { 2255 struct hns_roce_v2_priv *priv = hr_dev->priv; 2256 2257 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B) 2258 hns_roce_function_clear(hr_dev); 2259 2260 hns_roce_free_link_table(hr_dev, &priv->tpq); 2261 hns_roce_free_link_table(hr_dev, &priv->tsq); 2262 } 2263 2264 static int hns_roce_query_mbox_status(struct hns_roce_dev *hr_dev) 2265 { 2266 struct hns_roce_cmq_desc desc; 2267 struct hns_roce_mbox_status *mb_st = 2268 (struct hns_roce_mbox_status *)desc.data; 2269 enum hns_roce_cmd_return_status status; 2270 2271 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST, true); 2272 2273 status = hns_roce_cmq_send(hr_dev, &desc, 1); 2274 if (status) 2275 return status; 2276 2277 return le32_to_cpu(mb_st->mb_status_hw_run); 2278 } 2279 2280 static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev) 2281 { 2282 u32 status = hns_roce_query_mbox_status(hr_dev); 2283 2284 return status >> HNS_ROCE_HW_RUN_BIT_SHIFT; 2285 } 2286 2287 static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev) 2288 { 2289 u32 status = hns_roce_query_mbox_status(hr_dev); 2290 2291 return status & HNS_ROCE_HW_MB_STATUS_MASK; 2292 } 2293 2294 static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, u64 in_param, 2295 u64 out_param, u32 in_modifier, u8 op_modifier, 2296 u16 op, u16 token, int event) 2297 { 2298 struct hns_roce_cmq_desc desc; 2299 struct hns_roce_post_mbox *mb = (struct hns_roce_post_mbox *)desc.data; 2300 2301 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false); 2302 2303 mb->in_param_l = cpu_to_le32(in_param); 2304 mb->in_param_h = cpu_to_le32(in_param >> 32); 2305 mb->out_param_l = cpu_to_le32(out_param); 2306 mb->out_param_h = cpu_to_le32(out_param >> 32); 2307 mb->cmd_tag = cpu_to_le32(in_modifier << 8 | op); 2308 mb->token_event_en = cpu_to_le32(event << 16 | token); 2309 2310 return hns_roce_cmq_send(hr_dev, &desc, 1); 2311 } 2312 2313 static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param, 2314 u64 out_param, u32 in_modifier, u8 op_modifier, 2315 u16 op, u16 token, int event) 2316 { 2317 struct device *dev = hr_dev->dev; 2318 unsigned long end; 2319 int ret; 2320 2321 end = msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS) + jiffies; 2322 while (hns_roce_v2_cmd_pending(hr_dev)) { 2323 if (time_after(jiffies, end)) { 2324 dev_dbg(dev, "jiffies=%d end=%d\n", (int)jiffies, 2325 (int)end); 2326 return -EAGAIN; 2327 } 2328 cond_resched(); 2329 } 2330 2331 ret = hns_roce_mbox_post(hr_dev, in_param, out_param, in_modifier, 2332 op_modifier, op, token, event); 2333 if (ret) 2334 dev_err(dev, "Post mailbox fail(%d)\n", ret); 2335 2336 return ret; 2337 } 2338 2339 static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev, 2340 unsigned long timeout) 2341 { 2342 struct device *dev = hr_dev->dev; 2343 unsigned long end; 2344 u32 status; 2345 2346 end = msecs_to_jiffies(timeout) + jiffies; 2347 while (hns_roce_v2_cmd_pending(hr_dev) && time_before(jiffies, end)) 2348 cond_resched(); 2349 2350 if (hns_roce_v2_cmd_pending(hr_dev)) { 2351 dev_err(dev, "[cmd_poll]hw run cmd TIMEDOUT!\n"); 2352 return -ETIMEDOUT; 2353 } 2354 2355 status = hns_roce_v2_cmd_complete(hr_dev); 2356 if (status != 0x1) { 2357 if (status == CMD_RST_PRC_EBUSY) 2358 return status; 2359 2360 dev_err(dev, "mailbox status 0x%x!\n", status); 2361 return -EBUSY; 2362 } 2363 2364 return 0; 2365 } 2366 2367 static int hns_roce_config_sgid_table(struct hns_roce_dev *hr_dev, 2368 int gid_index, const union ib_gid *gid, 2369 enum hns_roce_sgid_type sgid_type) 2370 { 2371 struct hns_roce_cmq_desc desc; 2372 struct hns_roce_cfg_sgid_tb *sgid_tb = 2373 (struct hns_roce_cfg_sgid_tb *)desc.data; 2374 u32 *p; 2375 2376 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false); 2377 2378 roce_set_field(sgid_tb->table_idx_rsv, CFG_SGID_TB_TABLE_IDX_M, 2379 CFG_SGID_TB_TABLE_IDX_S, gid_index); 2380 roce_set_field(sgid_tb->vf_sgid_type_rsv, CFG_SGID_TB_VF_SGID_TYPE_M, 2381 CFG_SGID_TB_VF_SGID_TYPE_S, sgid_type); 2382 2383 p = (u32 *)&gid->raw[0]; 2384 sgid_tb->vf_sgid_l = cpu_to_le32(*p); 2385 2386 p = (u32 *)&gid->raw[4]; 2387 sgid_tb->vf_sgid_ml = cpu_to_le32(*p); 2388 2389 p = (u32 *)&gid->raw[8]; 2390 sgid_tb->vf_sgid_mh = cpu_to_le32(*p); 2391 2392 p = (u32 *)&gid->raw[0xc]; 2393 sgid_tb->vf_sgid_h = cpu_to_le32(*p); 2394 2395 return hns_roce_cmq_send(hr_dev, &desc, 1); 2396 } 2397 2398 static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port, 2399 int gid_index, const union ib_gid *gid, 2400 const struct ib_gid_attr *attr) 2401 { 2402 enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1; 2403 int ret; 2404 2405 if (!gid || !attr) 2406 return -EINVAL; 2407 2408 if (attr->gid_type == IB_GID_TYPE_ROCE) 2409 sgid_type = GID_TYPE_FLAG_ROCE_V1; 2410 2411 if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { 2412 if (ipv6_addr_v4mapped((void *)gid)) 2413 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4; 2414 else 2415 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6; 2416 } 2417 2418 ret = hns_roce_config_sgid_table(hr_dev, gid_index, gid, sgid_type); 2419 if (ret) 2420 ibdev_err(&hr_dev->ib_dev, 2421 "failed to configure sgid table, ret = %d!\n", 2422 ret); 2423 2424 return ret; 2425 } 2426 2427 static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, 2428 u8 *addr) 2429 { 2430 struct hns_roce_cmq_desc desc; 2431 struct hns_roce_cfg_smac_tb *smac_tb = 2432 (struct hns_roce_cfg_smac_tb *)desc.data; 2433 u16 reg_smac_h; 2434 u32 reg_smac_l; 2435 2436 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SMAC_TB, false); 2437 2438 reg_smac_l = *(u32 *)(&addr[0]); 2439 reg_smac_h = *(u16 *)(&addr[4]); 2440 2441 memset(smac_tb, 0, sizeof(*smac_tb)); 2442 roce_set_field(smac_tb->tb_idx_rsv, 2443 CFG_SMAC_TB_IDX_M, 2444 CFG_SMAC_TB_IDX_S, phy_port); 2445 roce_set_field(smac_tb->vf_smac_h_rsv, 2446 CFG_SMAC_TB_VF_SMAC_H_M, 2447 CFG_SMAC_TB_VF_SMAC_H_S, reg_smac_h); 2448 smac_tb->vf_smac_l = cpu_to_le32(reg_smac_l); 2449 2450 return hns_roce_cmq_send(hr_dev, &desc, 1); 2451 } 2452 2453 static int set_mtpt_pbl(struct hns_roce_v2_mpt_entry *mpt_entry, 2454 struct hns_roce_mr *mr) 2455 { 2456 struct sg_dma_page_iter sg_iter; 2457 u64 page_addr; 2458 u64 *pages; 2459 int i; 2460 2461 mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size); 2462 mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3)); 2463 roce_set_field(mpt_entry->byte_48_mode_ba, 2464 V2_MPT_BYTE_48_PBL_BA_H_M, V2_MPT_BYTE_48_PBL_BA_H_S, 2465 upper_32_bits(mr->pbl_ba >> 3)); 2466 2467 pages = (u64 *)__get_free_page(GFP_KERNEL); 2468 if (!pages) 2469 return -ENOMEM; 2470 2471 i = 0; 2472 for_each_sg_dma_page(mr->umem->sg_head.sgl, &sg_iter, mr->umem->nmap, 0) { 2473 page_addr = sg_page_iter_dma_address(&sg_iter); 2474 pages[i] = page_addr >> 6; 2475 2476 /* Record the first 2 entry directly to MTPT table */ 2477 if (i >= HNS_ROCE_V2_MAX_INNER_MTPT_NUM - 1) 2478 goto found; 2479 i++; 2480 } 2481 found: 2482 mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0])); 2483 roce_set_field(mpt_entry->byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M, 2484 V2_MPT_BYTE_56_PA0_H_S, upper_32_bits(pages[0])); 2485 2486 mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1])); 2487 roce_set_field(mpt_entry->byte_64_buf_pa1, V2_MPT_BYTE_64_PA1_H_M, 2488 V2_MPT_BYTE_64_PA1_H_S, upper_32_bits(pages[1])); 2489 roce_set_field(mpt_entry->byte_64_buf_pa1, 2490 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M, 2491 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S, 2492 mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET); 2493 2494 free_page((unsigned long)pages); 2495 2496 return 0; 2497 } 2498 2499 static int hns_roce_v2_write_mtpt(void *mb_buf, struct hns_roce_mr *mr, 2500 unsigned long mtpt_idx) 2501 { 2502 struct hns_roce_v2_mpt_entry *mpt_entry; 2503 int ret; 2504 2505 mpt_entry = mb_buf; 2506 memset(mpt_entry, 0, sizeof(*mpt_entry)); 2507 2508 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M, 2509 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID); 2510 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M, 2511 V2_MPT_BYTE_4_PBL_HOP_NUM_S, mr->pbl_hop_num == 2512 HNS_ROCE_HOP_NUM_0 ? 0 : mr->pbl_hop_num); 2513 roce_set_field(mpt_entry->byte_4_pd_hop_st, 2514 V2_MPT_BYTE_4_PBL_BA_PG_SZ_M, 2515 V2_MPT_BYTE_4_PBL_BA_PG_SZ_S, 2516 mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET); 2517 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M, 2518 V2_MPT_BYTE_4_PD_S, mr->pd); 2519 2520 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 0); 2521 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 0); 2522 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1); 2523 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_BIND_EN_S, 2524 (mr->access & IB_ACCESS_MW_BIND ? 1 : 0)); 2525 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_ATOMIC_EN_S, 2526 mr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0); 2527 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S, 2528 (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0)); 2529 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S, 2530 (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0)); 2531 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S, 2532 (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0)); 2533 2534 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 2535 mr->type == MR_TYPE_MR ? 0 : 1); 2536 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_INNER_PA_VLD_S, 2537 1); 2538 2539 mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size)); 2540 mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size)); 2541 mpt_entry->lkey = cpu_to_le32(mr->key); 2542 mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova)); 2543 mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova)); 2544 2545 if (mr->type == MR_TYPE_DMA) 2546 return 0; 2547 2548 ret = set_mtpt_pbl(mpt_entry, mr); 2549 2550 return ret; 2551 } 2552 2553 static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev, 2554 struct hns_roce_mr *mr, int flags, 2555 u32 pdn, int mr_access_flags, u64 iova, 2556 u64 size, void *mb_buf) 2557 { 2558 struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf; 2559 int ret = 0; 2560 2561 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M, 2562 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID); 2563 2564 if (flags & IB_MR_REREG_PD) { 2565 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M, 2566 V2_MPT_BYTE_4_PD_S, pdn); 2567 mr->pd = pdn; 2568 } 2569 2570 if (flags & IB_MR_REREG_ACCESS) { 2571 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, 2572 V2_MPT_BYTE_8_BIND_EN_S, 2573 (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0)); 2574 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, 2575 V2_MPT_BYTE_8_ATOMIC_EN_S, 2576 mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0); 2577 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S, 2578 mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0); 2579 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S, 2580 mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0); 2581 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S, 2582 mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0); 2583 } 2584 2585 if (flags & IB_MR_REREG_TRANS) { 2586 mpt_entry->va_l = cpu_to_le32(lower_32_bits(iova)); 2587 mpt_entry->va_h = cpu_to_le32(upper_32_bits(iova)); 2588 mpt_entry->len_l = cpu_to_le32(lower_32_bits(size)); 2589 mpt_entry->len_h = cpu_to_le32(upper_32_bits(size)); 2590 2591 mr->iova = iova; 2592 mr->size = size; 2593 2594 ret = set_mtpt_pbl(mpt_entry, mr); 2595 } 2596 2597 return ret; 2598 } 2599 2600 static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr) 2601 { 2602 struct hns_roce_v2_mpt_entry *mpt_entry; 2603 2604 mpt_entry = mb_buf; 2605 memset(mpt_entry, 0, sizeof(*mpt_entry)); 2606 2607 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M, 2608 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE); 2609 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M, 2610 V2_MPT_BYTE_4_PBL_HOP_NUM_S, 1); 2611 roce_set_field(mpt_entry->byte_4_pd_hop_st, 2612 V2_MPT_BYTE_4_PBL_BA_PG_SZ_M, 2613 V2_MPT_BYTE_4_PBL_BA_PG_SZ_S, 2614 mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET); 2615 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M, 2616 V2_MPT_BYTE_4_PD_S, mr->pd); 2617 2618 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 1); 2619 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1); 2620 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1); 2621 2622 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_FRE_S, 1); 2623 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0); 2624 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 0); 2625 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1); 2626 2627 mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size); 2628 2629 mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3)); 2630 roce_set_field(mpt_entry->byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M, 2631 V2_MPT_BYTE_48_PBL_BA_H_S, 2632 upper_32_bits(mr->pbl_ba >> 3)); 2633 2634 roce_set_field(mpt_entry->byte_64_buf_pa1, 2635 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M, 2636 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S, 2637 mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET); 2638 2639 return 0; 2640 } 2641 2642 static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw) 2643 { 2644 struct hns_roce_v2_mpt_entry *mpt_entry; 2645 2646 mpt_entry = mb_buf; 2647 memset(mpt_entry, 0, sizeof(*mpt_entry)); 2648 2649 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M, 2650 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE); 2651 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M, 2652 V2_MPT_BYTE_4_PD_S, mw->pdn); 2653 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M, 2654 V2_MPT_BYTE_4_PBL_HOP_NUM_S, 2655 mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : 2656 mw->pbl_hop_num); 2657 roce_set_field(mpt_entry->byte_4_pd_hop_st, 2658 V2_MPT_BYTE_4_PBL_BA_PG_SZ_M, 2659 V2_MPT_BYTE_4_PBL_BA_PG_SZ_S, 2660 mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET); 2661 2662 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1); 2663 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1); 2664 2665 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0); 2666 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 1); 2667 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1); 2668 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BQP_S, 2669 mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1); 2670 2671 roce_set_field(mpt_entry->byte_64_buf_pa1, 2672 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M, 2673 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S, 2674 mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET); 2675 2676 mpt_entry->lkey = cpu_to_le32(mw->rkey); 2677 2678 return 0; 2679 } 2680 2681 static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n) 2682 { 2683 return hns_roce_buf_offset(&hr_cq->buf, n * HNS_ROCE_V2_CQE_ENTRY_SIZE); 2684 } 2685 2686 static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, int n) 2687 { 2688 struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe); 2689 2690 /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */ 2691 return (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_OWNER_S) ^ 2692 !!(n & hr_cq->cq_depth)) ? cqe : NULL; 2693 } 2694 2695 static struct hns_roce_v2_cqe *next_cqe_sw_v2(struct hns_roce_cq *hr_cq) 2696 { 2697 return get_sw_cqe_v2(hr_cq, hr_cq->cons_index); 2698 } 2699 2700 static void *get_srq_wqe(struct hns_roce_srq *srq, int n) 2701 { 2702 return hns_roce_buf_offset(&srq->buf, n << srq->wqe_shift); 2703 } 2704 2705 static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index) 2706 { 2707 /* always called with interrupts disabled. */ 2708 spin_lock(&srq->lock); 2709 2710 bitmap_clear(srq->idx_que.bitmap, wqe_index, 1); 2711 srq->tail++; 2712 2713 spin_unlock(&srq->lock); 2714 } 2715 2716 static void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index) 2717 { 2718 *hr_cq->set_ci_db = cons_index & V2_CQ_DB_PARAMETER_CONS_IDX_M; 2719 } 2720 2721 static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, 2722 struct hns_roce_srq *srq) 2723 { 2724 struct hns_roce_v2_cqe *cqe, *dest; 2725 u32 prod_index; 2726 int nfreed = 0; 2727 int wqe_index; 2728 u8 owner_bit; 2729 2730 for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index); 2731 ++prod_index) { 2732 if (prod_index > hr_cq->cons_index + hr_cq->ib_cq.cqe) 2733 break; 2734 } 2735 2736 /* 2737 * Now backwards through the CQ, removing CQ entries 2738 * that match our QP by overwriting them with next entries. 2739 */ 2740 while ((int) --prod_index - (int) hr_cq->cons_index >= 0) { 2741 cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe); 2742 if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M, 2743 V2_CQE_BYTE_16_LCL_QPN_S) & 2744 HNS_ROCE_V2_CQE_QPN_MASK) == qpn) { 2745 if (srq && 2746 roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S)) { 2747 wqe_index = roce_get_field(cqe->byte_4, 2748 V2_CQE_BYTE_4_WQE_INDX_M, 2749 V2_CQE_BYTE_4_WQE_INDX_S); 2750 hns_roce_free_srq_wqe(srq, wqe_index); 2751 } 2752 ++nfreed; 2753 } else if (nfreed) { 2754 dest = get_cqe_v2(hr_cq, (prod_index + nfreed) & 2755 hr_cq->ib_cq.cqe); 2756 owner_bit = roce_get_bit(dest->byte_4, 2757 V2_CQE_BYTE_4_OWNER_S); 2758 memcpy(dest, cqe, sizeof(*cqe)); 2759 roce_set_bit(dest->byte_4, V2_CQE_BYTE_4_OWNER_S, 2760 owner_bit); 2761 } 2762 } 2763 2764 if (nfreed) { 2765 hr_cq->cons_index += nfreed; 2766 /* 2767 * Make sure update of buffer contents is done before 2768 * updating consumer index. 2769 */ 2770 wmb(); 2771 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index); 2772 } 2773 } 2774 2775 static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, 2776 struct hns_roce_srq *srq) 2777 { 2778 spin_lock_irq(&hr_cq->lock); 2779 __hns_roce_v2_cq_clean(hr_cq, qpn, srq); 2780 spin_unlock_irq(&hr_cq->lock); 2781 } 2782 2783 static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, 2784 struct hns_roce_cq *hr_cq, void *mb_buf, 2785 u64 *mtts, dma_addr_t dma_handle) 2786 { 2787 struct hns_roce_v2_cq_context *cq_context; 2788 2789 cq_context = mb_buf; 2790 memset(cq_context, 0, sizeof(*cq_context)); 2791 2792 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CQ_ST_M, 2793 V2_CQC_BYTE_4_CQ_ST_S, V2_CQ_STATE_VALID); 2794 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M, 2795 V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE); 2796 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M, 2797 V2_CQC_BYTE_4_SHIFT_S, ilog2(hr_cq->cq_depth)); 2798 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M, 2799 V2_CQC_BYTE_4_CEQN_S, hr_cq->vector); 2800 2801 roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQN_M, 2802 V2_CQC_BYTE_8_CQN_S, hr_cq->cqn); 2803 2804 cq_context->cqe_cur_blk_addr = cpu_to_le32(mtts[0] >> PAGE_ADDR_SHIFT); 2805 2806 roce_set_field(cq_context->byte_16_hop_addr, 2807 V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M, 2808 V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S, 2809 mtts[0] >> (32 + PAGE_ADDR_SHIFT)); 2810 roce_set_field(cq_context->byte_16_hop_addr, 2811 V2_CQC_BYTE_16_CQE_HOP_NUM_M, 2812 V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev->caps.cqe_hop_num == 2813 HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num); 2814 2815 cq_context->cqe_nxt_blk_addr = cpu_to_le32(mtts[1] >> PAGE_ADDR_SHIFT); 2816 roce_set_field(cq_context->byte_24_pgsz_addr, 2817 V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M, 2818 V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S, 2819 mtts[1] >> (32 + PAGE_ADDR_SHIFT)); 2820 roce_set_field(cq_context->byte_24_pgsz_addr, 2821 V2_CQC_BYTE_24_CQE_BA_PG_SZ_M, 2822 V2_CQC_BYTE_24_CQE_BA_PG_SZ_S, 2823 hr_dev->caps.cqe_ba_pg_sz + PG_SHIFT_OFFSET); 2824 roce_set_field(cq_context->byte_24_pgsz_addr, 2825 V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M, 2826 V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S, 2827 hr_dev->caps.cqe_buf_pg_sz + PG_SHIFT_OFFSET); 2828 2829 cq_context->cqe_ba = cpu_to_le32(dma_handle >> 3); 2830 2831 roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M, 2832 V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3))); 2833 2834 if (hr_cq->db_en) 2835 roce_set_bit(cq_context->byte_44_db_record, 2836 V2_CQC_BYTE_44_DB_RECORD_EN_S, 1); 2837 2838 roce_set_field(cq_context->byte_44_db_record, 2839 V2_CQC_BYTE_44_DB_RECORD_ADDR_M, 2840 V2_CQC_BYTE_44_DB_RECORD_ADDR_S, 2841 ((u32)hr_cq->db.dma) >> 1); 2842 cq_context->db_record_addr = cpu_to_le32(hr_cq->db.dma >> 32); 2843 2844 roce_set_field(cq_context->byte_56_cqe_period_maxcnt, 2845 V2_CQC_BYTE_56_CQ_MAX_CNT_M, 2846 V2_CQC_BYTE_56_CQ_MAX_CNT_S, 2847 HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM); 2848 roce_set_field(cq_context->byte_56_cqe_period_maxcnt, 2849 V2_CQC_BYTE_56_CQ_PERIOD_M, 2850 V2_CQC_BYTE_56_CQ_PERIOD_S, 2851 HNS_ROCE_V2_CQ_DEFAULT_INTERVAL); 2852 } 2853 2854 static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq, 2855 enum ib_cq_notify_flags flags) 2856 { 2857 struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); 2858 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); 2859 u32 notification_flag; 2860 __le32 doorbell[2]; 2861 2862 doorbell[0] = 0; 2863 doorbell[1] = 0; 2864 2865 notification_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? 2866 V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL; 2867 /* 2868 * flags = 0; Notification Flag = 1, next 2869 * flags = 1; Notification Flag = 0, solocited 2870 */ 2871 roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_TAG_M, V2_DB_BYTE_4_TAG_S, 2872 hr_cq->cqn); 2873 roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S, 2874 HNS_ROCE_V2_CQ_DB_NTR); 2875 roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M, 2876 V2_CQ_DB_PARAMETER_CONS_IDX_S, 2877 hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1)); 2878 roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M, 2879 V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq->arm_sn & 0x3); 2880 roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S, 2881 notification_flag); 2882 2883 hns_roce_write64(hr_dev, doorbell, hr_cq->cq_db_l); 2884 2885 return 0; 2886 } 2887 2888 static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe, 2889 struct hns_roce_qp **cur_qp, 2890 struct ib_wc *wc) 2891 { 2892 struct hns_roce_rinl_sge *sge_list; 2893 u32 wr_num, wr_cnt, sge_num; 2894 u32 sge_cnt, data_len, size; 2895 void *wqe_buf; 2896 2897 wr_num = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_WQE_INDX_M, 2898 V2_CQE_BYTE_4_WQE_INDX_S) & 0xffff; 2899 wr_cnt = wr_num & ((*cur_qp)->rq.wqe_cnt - 1); 2900 2901 sge_list = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sg_list; 2902 sge_num = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sge_cnt; 2903 wqe_buf = hns_roce_get_recv_wqe(*cur_qp, wr_cnt); 2904 data_len = wc->byte_len; 2905 2906 for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) { 2907 size = min(sge_list[sge_cnt].len, data_len); 2908 memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size); 2909 2910 data_len -= size; 2911 wqe_buf += size; 2912 } 2913 2914 if (data_len) { 2915 wc->status = IB_WC_LOC_LEN_ERR; 2916 return -EAGAIN; 2917 } 2918 2919 return 0; 2920 } 2921 2922 static int sw_comp(struct hns_roce_qp *hr_qp, struct hns_roce_wq *wq, 2923 int num_entries, struct ib_wc *wc) 2924 { 2925 unsigned int left; 2926 int npolled = 0; 2927 2928 left = wq->head - wq->tail; 2929 if (left == 0) 2930 return 0; 2931 2932 left = min_t(unsigned int, (unsigned int)num_entries, left); 2933 while (npolled < left) { 2934 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; 2935 wc->status = IB_WC_WR_FLUSH_ERR; 2936 wc->vendor_err = 0; 2937 wc->qp = &hr_qp->ibqp; 2938 2939 wq->tail++; 2940 wc++; 2941 npolled++; 2942 } 2943 2944 return npolled; 2945 } 2946 2947 static int hns_roce_v2_sw_poll_cq(struct hns_roce_cq *hr_cq, int num_entries, 2948 struct ib_wc *wc) 2949 { 2950 struct hns_roce_qp *hr_qp; 2951 int npolled = 0; 2952 2953 list_for_each_entry(hr_qp, &hr_cq->sq_list, sq_node) { 2954 npolled += sw_comp(hr_qp, &hr_qp->sq, 2955 num_entries - npolled, wc + npolled); 2956 if (npolled >= num_entries) 2957 goto out; 2958 } 2959 2960 list_for_each_entry(hr_qp, &hr_cq->rq_list, rq_node) { 2961 npolled += sw_comp(hr_qp, &hr_qp->rq, 2962 num_entries - npolled, wc + npolled); 2963 if (npolled >= num_entries) 2964 goto out; 2965 } 2966 2967 out: 2968 return npolled; 2969 } 2970 2971 static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, 2972 struct hns_roce_qp **cur_qp, struct ib_wc *wc) 2973 { 2974 struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); 2975 struct hns_roce_srq *srq = NULL; 2976 struct hns_roce_v2_cqe *cqe; 2977 struct hns_roce_qp *hr_qp; 2978 struct hns_roce_wq *wq; 2979 int is_send; 2980 u16 wqe_ctr; 2981 u32 opcode; 2982 u32 status; 2983 int qpn; 2984 int ret; 2985 2986 /* Find cqe according to consumer index */ 2987 cqe = next_cqe_sw_v2(hr_cq); 2988 if (!cqe) 2989 return -EAGAIN; 2990 2991 ++hr_cq->cons_index; 2992 /* Memory barrier */ 2993 rmb(); 2994 2995 /* 0->SQ, 1->RQ */ 2996 is_send = !roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S); 2997 2998 qpn = roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M, 2999 V2_CQE_BYTE_16_LCL_QPN_S); 3000 3001 if (!*cur_qp || (qpn & HNS_ROCE_V2_CQE_QPN_MASK) != (*cur_qp)->qpn) { 3002 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn); 3003 if (unlikely(!hr_qp)) { 3004 ibdev_err(&hr_dev->ib_dev, 3005 "CQ %06lx with entry for unknown QPN %06x\n", 3006 hr_cq->cqn, qpn & HNS_ROCE_V2_CQE_QPN_MASK); 3007 return -EINVAL; 3008 } 3009 *cur_qp = hr_qp; 3010 } 3011 3012 hr_qp = *cur_qp; 3013 wc->qp = &(*cur_qp)->ibqp; 3014 wc->vendor_err = 0; 3015 3016 if (is_send) { 3017 wq = &(*cur_qp)->sq; 3018 if ((*cur_qp)->sq_signal_bits) { 3019 /* 3020 * If sg_signal_bit is 1, 3021 * firstly tail pointer updated to wqe 3022 * which current cqe correspond to 3023 */ 3024 wqe_ctr = (u16)roce_get_field(cqe->byte_4, 3025 V2_CQE_BYTE_4_WQE_INDX_M, 3026 V2_CQE_BYTE_4_WQE_INDX_S); 3027 wq->tail += (wqe_ctr - (u16)wq->tail) & 3028 (wq->wqe_cnt - 1); 3029 } 3030 3031 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; 3032 ++wq->tail; 3033 } else if ((*cur_qp)->ibqp.srq) { 3034 srq = to_hr_srq((*cur_qp)->ibqp.srq); 3035 wqe_ctr = (u16)roce_get_field(cqe->byte_4, 3036 V2_CQE_BYTE_4_WQE_INDX_M, 3037 V2_CQE_BYTE_4_WQE_INDX_S); 3038 wc->wr_id = srq->wrid[wqe_ctr]; 3039 hns_roce_free_srq_wqe(srq, wqe_ctr); 3040 } else { 3041 /* Update tail pointer, record wr_id */ 3042 wq = &(*cur_qp)->rq; 3043 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; 3044 ++wq->tail; 3045 } 3046 3047 status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M, 3048 V2_CQE_BYTE_4_STATUS_S); 3049 switch (status & HNS_ROCE_V2_CQE_STATUS_MASK) { 3050 case HNS_ROCE_CQE_V2_SUCCESS: 3051 wc->status = IB_WC_SUCCESS; 3052 break; 3053 case HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR: 3054 wc->status = IB_WC_LOC_LEN_ERR; 3055 break; 3056 case HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR: 3057 wc->status = IB_WC_LOC_QP_OP_ERR; 3058 break; 3059 case HNS_ROCE_CQE_V2_LOCAL_PROT_ERR: 3060 wc->status = IB_WC_LOC_PROT_ERR; 3061 break; 3062 case HNS_ROCE_CQE_V2_WR_FLUSH_ERR: 3063 wc->status = IB_WC_WR_FLUSH_ERR; 3064 break; 3065 case HNS_ROCE_CQE_V2_MW_BIND_ERR: 3066 wc->status = IB_WC_MW_BIND_ERR; 3067 break; 3068 case HNS_ROCE_CQE_V2_BAD_RESP_ERR: 3069 wc->status = IB_WC_BAD_RESP_ERR; 3070 break; 3071 case HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR: 3072 wc->status = IB_WC_LOC_ACCESS_ERR; 3073 break; 3074 case HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR: 3075 wc->status = IB_WC_REM_INV_REQ_ERR; 3076 break; 3077 case HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR: 3078 wc->status = IB_WC_REM_ACCESS_ERR; 3079 break; 3080 case HNS_ROCE_CQE_V2_REMOTE_OP_ERR: 3081 wc->status = IB_WC_REM_OP_ERR; 3082 break; 3083 case HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR: 3084 wc->status = IB_WC_RETRY_EXC_ERR; 3085 break; 3086 case HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR: 3087 wc->status = IB_WC_RNR_RETRY_EXC_ERR; 3088 break; 3089 case HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR: 3090 wc->status = IB_WC_REM_ABORT_ERR; 3091 break; 3092 default: 3093 wc->status = IB_WC_GENERAL_ERR; 3094 break; 3095 } 3096 3097 /* 3098 * Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state gets 3099 * into errored mode. Hence, as a workaround to this hardware 3100 * limitation, driver needs to assist in flushing. But the flushing 3101 * operation uses mailbox to convey the QP state to the hardware and 3102 * which can sleep due to the mutex protection around the mailbox calls. 3103 * Hence, use the deferred flush for now. Once wc error detected, the 3104 * flushing operation is needed. 3105 */ 3106 if (wc->status != IB_WC_SUCCESS && 3107 wc->status != IB_WC_WR_FLUSH_ERR) { 3108 ibdev_err(&hr_dev->ib_dev, "error cqe status is: 0x%x\n", 3109 status & HNS_ROCE_V2_CQE_STATUS_MASK); 3110 3111 if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &hr_qp->flush_flag)) 3112 init_flush_work(hr_dev, hr_qp); 3113 3114 return 0; 3115 } 3116 3117 if (wc->status == IB_WC_WR_FLUSH_ERR) 3118 return 0; 3119 3120 if (is_send) { 3121 wc->wc_flags = 0; 3122 /* SQ corresponding to CQE */ 3123 switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M, 3124 V2_CQE_BYTE_4_OPCODE_S) & 0x1f) { 3125 case HNS_ROCE_SQ_OPCODE_SEND: 3126 wc->opcode = IB_WC_SEND; 3127 break; 3128 case HNS_ROCE_SQ_OPCODE_SEND_WITH_INV: 3129 wc->opcode = IB_WC_SEND; 3130 break; 3131 case HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM: 3132 wc->opcode = IB_WC_SEND; 3133 wc->wc_flags |= IB_WC_WITH_IMM; 3134 break; 3135 case HNS_ROCE_SQ_OPCODE_RDMA_READ: 3136 wc->opcode = IB_WC_RDMA_READ; 3137 wc->byte_len = le32_to_cpu(cqe->byte_cnt); 3138 break; 3139 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE: 3140 wc->opcode = IB_WC_RDMA_WRITE; 3141 break; 3142 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM: 3143 wc->opcode = IB_WC_RDMA_WRITE; 3144 wc->wc_flags |= IB_WC_WITH_IMM; 3145 break; 3146 case HNS_ROCE_SQ_OPCODE_LOCAL_INV: 3147 wc->opcode = IB_WC_LOCAL_INV; 3148 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 3149 break; 3150 case HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP: 3151 wc->opcode = IB_WC_COMP_SWAP; 3152 wc->byte_len = 8; 3153 break; 3154 case HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD: 3155 wc->opcode = IB_WC_FETCH_ADD; 3156 wc->byte_len = 8; 3157 break; 3158 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP: 3159 wc->opcode = IB_WC_MASKED_COMP_SWAP; 3160 wc->byte_len = 8; 3161 break; 3162 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD: 3163 wc->opcode = IB_WC_MASKED_FETCH_ADD; 3164 wc->byte_len = 8; 3165 break; 3166 case HNS_ROCE_SQ_OPCODE_FAST_REG_WR: 3167 wc->opcode = IB_WC_REG_MR; 3168 break; 3169 case HNS_ROCE_SQ_OPCODE_BIND_MW: 3170 wc->opcode = IB_WC_REG_MR; 3171 break; 3172 default: 3173 wc->status = IB_WC_GENERAL_ERR; 3174 break; 3175 } 3176 } else { 3177 /* RQ correspond to CQE */ 3178 wc->byte_len = le32_to_cpu(cqe->byte_cnt); 3179 3180 opcode = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M, 3181 V2_CQE_BYTE_4_OPCODE_S); 3182 switch (opcode & 0x1f) { 3183 case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM: 3184 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 3185 wc->wc_flags = IB_WC_WITH_IMM; 3186 wc->ex.imm_data = 3187 cpu_to_be32(le32_to_cpu(cqe->immtdata)); 3188 break; 3189 case HNS_ROCE_V2_OPCODE_SEND: 3190 wc->opcode = IB_WC_RECV; 3191 wc->wc_flags = 0; 3192 break; 3193 case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM: 3194 wc->opcode = IB_WC_RECV; 3195 wc->wc_flags = IB_WC_WITH_IMM; 3196 wc->ex.imm_data = 3197 cpu_to_be32(le32_to_cpu(cqe->immtdata)); 3198 break; 3199 case HNS_ROCE_V2_OPCODE_SEND_WITH_INV: 3200 wc->opcode = IB_WC_RECV; 3201 wc->wc_flags = IB_WC_WITH_INVALIDATE; 3202 wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey); 3203 break; 3204 default: 3205 wc->status = IB_WC_GENERAL_ERR; 3206 break; 3207 } 3208 3209 if ((wc->qp->qp_type == IB_QPT_RC || 3210 wc->qp->qp_type == IB_QPT_UC) && 3211 (opcode == HNS_ROCE_V2_OPCODE_SEND || 3212 opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM || 3213 opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) && 3214 (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_RQ_INLINE_S))) { 3215 ret = hns_roce_handle_recv_inl_wqe(cqe, cur_qp, wc); 3216 if (ret) 3217 return -EAGAIN; 3218 } 3219 3220 wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M, 3221 V2_CQE_BYTE_32_SL_S); 3222 wc->src_qp = (u8)roce_get_field(cqe->byte_32, 3223 V2_CQE_BYTE_32_RMT_QPN_M, 3224 V2_CQE_BYTE_32_RMT_QPN_S); 3225 wc->slid = 0; 3226 wc->wc_flags |= (roce_get_bit(cqe->byte_32, 3227 V2_CQE_BYTE_32_GRH_S) ? 3228 IB_WC_GRH : 0); 3229 wc->port_num = roce_get_field(cqe->byte_32, 3230 V2_CQE_BYTE_32_PORTN_M, V2_CQE_BYTE_32_PORTN_S); 3231 wc->pkey_index = 0; 3232 3233 if (roce_get_bit(cqe->byte_28, V2_CQE_BYTE_28_VID_VLD_S)) { 3234 wc->vlan_id = (u16)roce_get_field(cqe->byte_28, 3235 V2_CQE_BYTE_28_VID_M, 3236 V2_CQE_BYTE_28_VID_S); 3237 wc->wc_flags |= IB_WC_WITH_VLAN; 3238 } else { 3239 wc->vlan_id = 0xffff; 3240 } 3241 3242 wc->network_hdr_type = roce_get_field(cqe->byte_28, 3243 V2_CQE_BYTE_28_PORT_TYPE_M, 3244 V2_CQE_BYTE_28_PORT_TYPE_S); 3245 } 3246 3247 return 0; 3248 } 3249 3250 static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries, 3251 struct ib_wc *wc) 3252 { 3253 struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); 3254 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); 3255 struct hns_roce_qp *cur_qp = NULL; 3256 unsigned long flags; 3257 int npolled; 3258 3259 spin_lock_irqsave(&hr_cq->lock, flags); 3260 3261 /* 3262 * When the device starts to reset, the state is RST_DOWN. At this time, 3263 * there may still be some valid CQEs in the hardware that are not 3264 * polled. Therefore, it is not allowed to switch to the software mode 3265 * immediately. When the state changes to UNINIT, CQE no longer exists 3266 * in the hardware, and then switch to software mode. 3267 */ 3268 if (hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT) { 3269 npolled = hns_roce_v2_sw_poll_cq(hr_cq, num_entries, wc); 3270 goto out; 3271 } 3272 3273 for (npolled = 0; npolled < num_entries; ++npolled) { 3274 if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled)) 3275 break; 3276 } 3277 3278 if (npolled) { 3279 /* Memory barrier */ 3280 wmb(); 3281 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index); 3282 } 3283 3284 out: 3285 spin_unlock_irqrestore(&hr_cq->lock, flags); 3286 3287 return npolled; 3288 } 3289 3290 static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type, 3291 int step_idx) 3292 { 3293 int op; 3294 3295 if (type == HEM_TYPE_SCCC && step_idx) 3296 return -EINVAL; 3297 3298 switch (type) { 3299 case HEM_TYPE_QPC: 3300 op = HNS_ROCE_CMD_WRITE_QPC_BT0; 3301 break; 3302 case HEM_TYPE_MTPT: 3303 op = HNS_ROCE_CMD_WRITE_MPT_BT0; 3304 break; 3305 case HEM_TYPE_CQC: 3306 op = HNS_ROCE_CMD_WRITE_CQC_BT0; 3307 break; 3308 case HEM_TYPE_SRQC: 3309 op = HNS_ROCE_CMD_WRITE_SRQC_BT0; 3310 break; 3311 case HEM_TYPE_SCCC: 3312 op = HNS_ROCE_CMD_WRITE_SCCC_BT0; 3313 break; 3314 case HEM_TYPE_QPC_TIMER: 3315 op = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0; 3316 break; 3317 case HEM_TYPE_CQC_TIMER: 3318 op = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0; 3319 break; 3320 default: 3321 dev_warn(hr_dev->dev, 3322 "Table %d not to be written by mailbox!\n", type); 3323 return -EINVAL; 3324 } 3325 3326 return op + step_idx; 3327 } 3328 3329 static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev, 3330 struct hns_roce_hem_table *table, int obj, 3331 int step_idx) 3332 { 3333 struct hns_roce_cmd_mailbox *mailbox; 3334 struct hns_roce_hem_iter iter; 3335 struct hns_roce_hem_mhop mhop; 3336 struct hns_roce_hem *hem; 3337 unsigned long mhop_obj = obj; 3338 int i, j, k; 3339 int ret = 0; 3340 u64 hem_idx = 0; 3341 u64 l1_idx = 0; 3342 u64 bt_ba = 0; 3343 u32 chunk_ba_num; 3344 u32 hop_num; 3345 int op; 3346 3347 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) 3348 return 0; 3349 3350 hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); 3351 i = mhop.l0_idx; 3352 j = mhop.l1_idx; 3353 k = mhop.l2_idx; 3354 hop_num = mhop.hop_num; 3355 chunk_ba_num = mhop.bt_chunk_size / 8; 3356 3357 if (hop_num == 2) { 3358 hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num + 3359 k; 3360 l1_idx = i * chunk_ba_num + j; 3361 } else if (hop_num == 1) { 3362 hem_idx = i * chunk_ba_num + j; 3363 } else if (hop_num == HNS_ROCE_HOP_NUM_0) { 3364 hem_idx = i; 3365 } 3366 3367 op = get_op_for_set_hem(hr_dev, table->type, step_idx); 3368 if (op == -EINVAL) 3369 return 0; 3370 3371 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 3372 if (IS_ERR(mailbox)) 3373 return PTR_ERR(mailbox); 3374 3375 if (table->type == HEM_TYPE_SCCC) 3376 obj = mhop.l0_idx; 3377 3378 if (check_whether_last_step(hop_num, step_idx)) { 3379 hem = table->hem[hem_idx]; 3380 for (hns_roce_hem_first(hem, &iter); 3381 !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) { 3382 bt_ba = hns_roce_hem_addr(&iter); 3383 3384 /* configure the ba, tag, and op */ 3385 ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, 3386 obj, 0, op, 3387 HNS_ROCE_CMD_TIMEOUT_MSECS); 3388 } 3389 } else { 3390 if (step_idx == 0) 3391 bt_ba = table->bt_l0_dma_addr[i]; 3392 else if (step_idx == 1 && hop_num == 2) 3393 bt_ba = table->bt_l1_dma_addr[l1_idx]; 3394 3395 /* configure the ba, tag, and op */ 3396 ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj, 3397 0, op, HNS_ROCE_CMD_TIMEOUT_MSECS); 3398 } 3399 3400 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 3401 return ret; 3402 } 3403 3404 static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev, 3405 struct hns_roce_hem_table *table, int obj, 3406 int step_idx) 3407 { 3408 struct device *dev = hr_dev->dev; 3409 struct hns_roce_cmd_mailbox *mailbox; 3410 int ret; 3411 u16 op = 0xff; 3412 3413 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) 3414 return 0; 3415 3416 switch (table->type) { 3417 case HEM_TYPE_QPC: 3418 op = HNS_ROCE_CMD_DESTROY_QPC_BT0; 3419 break; 3420 case HEM_TYPE_MTPT: 3421 op = HNS_ROCE_CMD_DESTROY_MPT_BT0; 3422 break; 3423 case HEM_TYPE_CQC: 3424 op = HNS_ROCE_CMD_DESTROY_CQC_BT0; 3425 break; 3426 case HEM_TYPE_SCCC: 3427 case HEM_TYPE_QPC_TIMER: 3428 case HEM_TYPE_CQC_TIMER: 3429 break; 3430 case HEM_TYPE_SRQC: 3431 op = HNS_ROCE_CMD_DESTROY_SRQC_BT0; 3432 break; 3433 default: 3434 dev_warn(dev, "Table %d not to be destroyed by mailbox!\n", 3435 table->type); 3436 return 0; 3437 } 3438 3439 if (table->type == HEM_TYPE_SCCC || 3440 table->type == HEM_TYPE_QPC_TIMER || 3441 table->type == HEM_TYPE_CQC_TIMER) 3442 return 0; 3443 3444 op += step_idx; 3445 3446 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 3447 if (IS_ERR(mailbox)) 3448 return PTR_ERR(mailbox); 3449 3450 /* configure the tag and op */ 3451 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, obj, 0, op, 3452 HNS_ROCE_CMD_TIMEOUT_MSECS); 3453 3454 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 3455 return ret; 3456 } 3457 3458 static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev, 3459 struct hns_roce_v2_qp_context *context, 3460 struct hns_roce_qp *hr_qp) 3461 { 3462 struct hns_roce_cmd_mailbox *mailbox; 3463 int ret; 3464 3465 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 3466 if (IS_ERR(mailbox)) 3467 return PTR_ERR(mailbox); 3468 3469 memcpy(mailbox->buf, context, sizeof(*context) * 2); 3470 3471 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0, 3472 HNS_ROCE_CMD_MODIFY_QPC, 3473 HNS_ROCE_CMD_TIMEOUT_MSECS); 3474 3475 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 3476 3477 return ret; 3478 } 3479 3480 static void set_access_flags(struct hns_roce_qp *hr_qp, 3481 struct hns_roce_v2_qp_context *context, 3482 struct hns_roce_v2_qp_context *qpc_mask, 3483 const struct ib_qp_attr *attr, int attr_mask) 3484 { 3485 u8 dest_rd_atomic; 3486 u32 access_flags; 3487 3488 dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ? 3489 attr->max_dest_rd_atomic : hr_qp->resp_depth; 3490 3491 access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ? 3492 attr->qp_access_flags : hr_qp->atomic_rd_en; 3493 3494 if (!dest_rd_atomic) 3495 access_flags &= IB_ACCESS_REMOTE_WRITE; 3496 3497 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 3498 !!(access_flags & IB_ACCESS_REMOTE_READ)); 3499 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 0); 3500 3501 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 3502 !!(access_flags & IB_ACCESS_REMOTE_WRITE)); 3503 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 0); 3504 3505 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 3506 !!(access_flags & IB_ACCESS_REMOTE_ATOMIC)); 3507 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0); 3508 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_EXT_ATE_S, 3509 !!(access_flags & IB_ACCESS_REMOTE_ATOMIC)); 3510 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_EXT_ATE_S, 0); 3511 } 3512 3513 static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp, 3514 struct hns_roce_v2_qp_context *context, 3515 struct hns_roce_v2_qp_context *qpc_mask) 3516 { 3517 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) 3518 roce_set_field(context->byte_4_sqpn_tst, 3519 V2_QPC_BYTE_4_SGE_SHIFT_M, 3520 V2_QPC_BYTE_4_SGE_SHIFT_S, 3521 ilog2((unsigned int)hr_qp->sge.sge_cnt)); 3522 else 3523 roce_set_field(context->byte_4_sqpn_tst, 3524 V2_QPC_BYTE_4_SGE_SHIFT_M, 3525 V2_QPC_BYTE_4_SGE_SHIFT_S, 3526 hr_qp->sq.max_gs > 3527 HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE ? 3528 ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0); 3529 3530 roce_set_field(context->byte_20_smac_sgid_idx, 3531 V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 3532 ilog2((unsigned int)hr_qp->sq.wqe_cnt)); 3533 3534 roce_set_field(context->byte_20_smac_sgid_idx, 3535 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 3536 (hr_qp->ibqp.qp_type == IB_QPT_XRC_INI || 3537 hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT || 3538 hr_qp->ibqp.srq) ? 0 : 3539 ilog2((unsigned int)hr_qp->rq.wqe_cnt)); 3540 } 3541 3542 static void modify_qp_reset_to_init(struct ib_qp *ibqp, 3543 const struct ib_qp_attr *attr, 3544 int attr_mask, 3545 struct hns_roce_v2_qp_context *context, 3546 struct hns_roce_v2_qp_context *qpc_mask) 3547 { 3548 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 3549 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 3550 3551 /* 3552 * In v2 engine, software pass context and context mask to hardware 3553 * when modifying qp. If software need modify some fields in context, 3554 * we should set all bits of the relevant fields in context mask to 3555 * 0 at the same time, else set them to 0x1. 3556 */ 3557 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M, 3558 V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type)); 3559 3560 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M, 3561 V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn); 3562 3563 roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M, 3564 V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn); 3565 3566 roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M, 3567 V2_QPC_BYTE_20_RQWS_S, ilog2(hr_qp->rq.max_gs)); 3568 3569 set_qpc_wqe_cnt(hr_qp, context, qpc_mask); 3570 3571 /* No VLAN need to set 0xFFF */ 3572 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M, 3573 V2_QPC_BYTE_24_VLAN_ID_S, 0xfff); 3574 3575 if (hr_qp->rdb_en) 3576 roce_set_bit(context->byte_68_rq_db, 3577 V2_QPC_BYTE_68_RQ_RECORD_EN_S, 1); 3578 3579 roce_set_field(context->byte_68_rq_db, 3580 V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M, 3581 V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S, 3582 ((u32)hr_qp->rdb.dma) >> 1); 3583 context->rq_db_record_addr = cpu_to_le32(hr_qp->rdb.dma >> 32); 3584 3585 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 3586 (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) ? 1 : 0); 3587 3588 roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M, 3589 V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn); 3590 if (ibqp->srq) { 3591 roce_set_field(context->byte_76_srqn_op_en, 3592 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 3593 to_hr_srq(ibqp->srq)->srqn); 3594 roce_set_bit(context->byte_76_srqn_op_en, 3595 V2_QPC_BYTE_76_SRQ_EN_S, 1); 3596 } 3597 3598 roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M, 3599 V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 4); 3600 3601 roce_set_bit(context->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 1); 3602 3603 hr_qp->access_flags = attr->qp_access_flags; 3604 roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M, 3605 V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn); 3606 } 3607 3608 static void modify_qp_init_to_init(struct ib_qp *ibqp, 3609 const struct ib_qp_attr *attr, int attr_mask, 3610 struct hns_roce_v2_qp_context *context, 3611 struct hns_roce_v2_qp_context *qpc_mask) 3612 { 3613 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 3614 3615 /* 3616 * In v2 engine, software pass context and context mask to hardware 3617 * when modifying qp. If software need modify some fields in context, 3618 * we should set all bits of the relevant fields in context mask to 3619 * 0 at the same time, else set them to 0x1. 3620 */ 3621 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M, 3622 V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type)); 3623 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M, 3624 V2_QPC_BYTE_4_TST_S, 0); 3625 3626 if (attr_mask & IB_QP_ACCESS_FLAGS) { 3627 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 3628 !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ)); 3629 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 3630 0); 3631 3632 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 3633 !!(attr->qp_access_flags & 3634 IB_ACCESS_REMOTE_WRITE)); 3635 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 3636 0); 3637 3638 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 3639 !!(attr->qp_access_flags & 3640 IB_ACCESS_REMOTE_ATOMIC)); 3641 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 3642 0); 3643 roce_set_bit(context->byte_76_srqn_op_en, 3644 V2_QPC_BYTE_76_EXT_ATE_S, 3645 !!(attr->qp_access_flags & 3646 IB_ACCESS_REMOTE_ATOMIC)); 3647 roce_set_bit(qpc_mask->byte_76_srqn_op_en, 3648 V2_QPC_BYTE_76_EXT_ATE_S, 0); 3649 } else { 3650 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 3651 !!(hr_qp->access_flags & IB_ACCESS_REMOTE_READ)); 3652 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 3653 0); 3654 3655 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 3656 !!(hr_qp->access_flags & IB_ACCESS_REMOTE_WRITE)); 3657 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 3658 0); 3659 3660 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 3661 !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC)); 3662 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 3663 0); 3664 roce_set_bit(context->byte_76_srqn_op_en, 3665 V2_QPC_BYTE_76_EXT_ATE_S, 3666 !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC)); 3667 roce_set_bit(qpc_mask->byte_76_srqn_op_en, 3668 V2_QPC_BYTE_76_EXT_ATE_S, 0); 3669 } 3670 3671 roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M, 3672 V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn); 3673 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M, 3674 V2_QPC_BYTE_16_PD_S, 0); 3675 3676 roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M, 3677 V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn); 3678 roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M, 3679 V2_QPC_BYTE_80_RX_CQN_S, 0); 3680 3681 roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M, 3682 V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn); 3683 roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M, 3684 V2_QPC_BYTE_252_TX_CQN_S, 0); 3685 3686 if (ibqp->srq) { 3687 roce_set_bit(context->byte_76_srqn_op_en, 3688 V2_QPC_BYTE_76_SRQ_EN_S, 1); 3689 roce_set_bit(qpc_mask->byte_76_srqn_op_en, 3690 V2_QPC_BYTE_76_SRQ_EN_S, 0); 3691 roce_set_field(context->byte_76_srqn_op_en, 3692 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 3693 to_hr_srq(ibqp->srq)->srqn); 3694 roce_set_field(qpc_mask->byte_76_srqn_op_en, 3695 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0); 3696 } 3697 3698 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M, 3699 V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn); 3700 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M, 3701 V2_QPC_BYTE_4_SQPN_S, 0); 3702 3703 if (attr_mask & IB_QP_DEST_QPN) { 3704 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M, 3705 V2_QPC_BYTE_56_DQPN_S, hr_qp->qpn); 3706 roce_set_field(qpc_mask->byte_56_dqpn_err, 3707 V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0); 3708 } 3709 } 3710 3711 static bool check_wqe_rq_mtt_count(struct hns_roce_dev *hr_dev, 3712 struct hns_roce_qp *hr_qp, int mtt_cnt, 3713 u32 page_size) 3714 { 3715 struct ib_device *ibdev = &hr_dev->ib_dev; 3716 3717 if (hr_qp->rq.wqe_cnt < 1) 3718 return true; 3719 3720 if (mtt_cnt < 1) { 3721 ibdev_err(ibdev, "failed to find RQWQE buf ba of QP(0x%lx)\n", 3722 hr_qp->qpn); 3723 return false; 3724 } 3725 3726 if (mtt_cnt < MTT_MIN_COUNT && 3727 (hr_qp->rq.offset + page_size) < hr_qp->buff_size) { 3728 ibdev_err(ibdev, 3729 "failed to find next RQWQE buf ba of QP(0x%lx)\n", 3730 hr_qp->qpn); 3731 return false; 3732 } 3733 3734 return true; 3735 } 3736 3737 static int modify_qp_init_to_rtr(struct ib_qp *ibqp, 3738 const struct ib_qp_attr *attr, int attr_mask, 3739 struct hns_roce_v2_qp_context *context, 3740 struct hns_roce_v2_qp_context *qpc_mask) 3741 { 3742 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 3743 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 3744 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 3745 struct ib_device *ibdev = &hr_dev->ib_dev; 3746 u64 mtts[MTT_MIN_COUNT] = { 0 }; 3747 dma_addr_t dma_handle_3; 3748 dma_addr_t dma_handle_2; 3749 u64 wqe_sge_ba; 3750 u32 page_size; 3751 u8 port_num; 3752 u64 *mtts_3; 3753 u64 *mtts_2; 3754 int count; 3755 u8 *dmac; 3756 u8 *smac; 3757 int port; 3758 3759 /* Search qp buf's mtts */ 3760 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); 3761 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 3762 hr_qp->rq.offset / page_size, mtts, 3763 MTT_MIN_COUNT, &wqe_sge_ba); 3764 if (!ibqp->srq) 3765 if (!check_wqe_rq_mtt_count(hr_dev, hr_qp, count, page_size)) 3766 return -EINVAL; 3767 3768 /* Search IRRL's mtts */ 3769 mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table, 3770 hr_qp->qpn, &dma_handle_2); 3771 if (!mtts_2) { 3772 ibdev_err(ibdev, "failed to find QP irrl_table\n"); 3773 return -EINVAL; 3774 } 3775 3776 /* Search TRRL's mtts */ 3777 mtts_3 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table, 3778 hr_qp->qpn, &dma_handle_3); 3779 if (!mtts_3) { 3780 ibdev_err(ibdev, "failed to find QP trrl_table\n"); 3781 return -EINVAL; 3782 } 3783 3784 if (attr_mask & IB_QP_ALT_PATH) { 3785 ibdev_err(ibdev, "INIT2RTR attr_mask (0x%x) error\n", 3786 attr_mask); 3787 return -EINVAL; 3788 } 3789 3790 dmac = (u8 *)attr->ah_attr.roce.dmac; 3791 context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3); 3792 qpc_mask->wqe_sge_ba = 0; 3793 3794 /* 3795 * In v2 engine, software pass context and context mask to hardware 3796 * when modifying qp. If software need modify some fields in context, 3797 * we should set all bits of the relevant fields in context mask to 3798 * 0 at the same time, else set them to 0x1. 3799 */ 3800 roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M, 3801 V2_QPC_BYTE_12_WQE_SGE_BA_S, wqe_sge_ba >> (32 + 3)); 3802 roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M, 3803 V2_QPC_BYTE_12_WQE_SGE_BA_S, 0); 3804 3805 roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M, 3806 V2_QPC_BYTE_12_SQ_HOP_NUM_S, 3807 hr_dev->caps.wqe_sq_hop_num == HNS_ROCE_HOP_NUM_0 ? 3808 0 : hr_dev->caps.wqe_sq_hop_num); 3809 roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M, 3810 V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0); 3811 3812 roce_set_field(context->byte_20_smac_sgid_idx, 3813 V2_QPC_BYTE_20_SGE_HOP_NUM_M, 3814 V2_QPC_BYTE_20_SGE_HOP_NUM_S, 3815 ((ibqp->qp_type == IB_QPT_GSI) || 3816 hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ? 3817 hr_dev->caps.wqe_sge_hop_num : 0); 3818 roce_set_field(qpc_mask->byte_20_smac_sgid_idx, 3819 V2_QPC_BYTE_20_SGE_HOP_NUM_M, 3820 V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0); 3821 3822 roce_set_field(context->byte_20_smac_sgid_idx, 3823 V2_QPC_BYTE_20_RQ_HOP_NUM_M, 3824 V2_QPC_BYTE_20_RQ_HOP_NUM_S, 3825 hr_dev->caps.wqe_rq_hop_num == HNS_ROCE_HOP_NUM_0 ? 3826 0 : hr_dev->caps.wqe_rq_hop_num); 3827 roce_set_field(qpc_mask->byte_20_smac_sgid_idx, 3828 V2_QPC_BYTE_20_RQ_HOP_NUM_M, 3829 V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0); 3830 3831 roce_set_field(context->byte_16_buf_ba_pg_sz, 3832 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M, 3833 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 3834 hr_qp->wqe_bt_pg_shift + PG_SHIFT_OFFSET); 3835 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, 3836 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M, 3837 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0); 3838 3839 roce_set_field(context->byte_16_buf_ba_pg_sz, 3840 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M, 3841 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 3842 hr_dev->caps.mtt_buf_pg_sz + PG_SHIFT_OFFSET); 3843 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, 3844 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M, 3845 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0); 3846 3847 context->rq_cur_blk_addr = cpu_to_le32(mtts[0] >> PAGE_ADDR_SHIFT); 3848 qpc_mask->rq_cur_blk_addr = 0; 3849 3850 roce_set_field(context->byte_92_srq_info, 3851 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M, 3852 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 3853 mtts[0] >> (32 + PAGE_ADDR_SHIFT)); 3854 roce_set_field(qpc_mask->byte_92_srq_info, 3855 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M, 3856 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0); 3857 3858 context->rq_nxt_blk_addr = cpu_to_le32(mtts[1] >> PAGE_ADDR_SHIFT); 3859 qpc_mask->rq_nxt_blk_addr = 0; 3860 3861 roce_set_field(context->byte_104_rq_sge, 3862 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M, 3863 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 3864 mtts[1] >> (32 + PAGE_ADDR_SHIFT)); 3865 roce_set_field(qpc_mask->byte_104_rq_sge, 3866 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M, 3867 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0); 3868 3869 roce_set_field(context->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M, 3870 V2_QPC_BYTE_132_TRRL_BA_S, dma_handle_3 >> 4); 3871 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M, 3872 V2_QPC_BYTE_132_TRRL_BA_S, 0); 3873 context->trrl_ba = cpu_to_le32(dma_handle_3 >> (16 + 4)); 3874 qpc_mask->trrl_ba = 0; 3875 roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M, 3876 V2_QPC_BYTE_140_TRRL_BA_S, 3877 (u32)(dma_handle_3 >> (32 + 16 + 4))); 3878 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M, 3879 V2_QPC_BYTE_140_TRRL_BA_S, 0); 3880 3881 context->irrl_ba = cpu_to_le32(dma_handle_2 >> 6); 3882 qpc_mask->irrl_ba = 0; 3883 roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M, 3884 V2_QPC_BYTE_208_IRRL_BA_S, 3885 dma_handle_2 >> (32 + 6)); 3886 roce_set_field(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M, 3887 V2_QPC_BYTE_208_IRRL_BA_S, 0); 3888 3889 roce_set_bit(context->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 1); 3890 roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 0); 3891 3892 roce_set_bit(context->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S, 3893 hr_qp->sq_signal_bits); 3894 roce_set_bit(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S, 3895 0); 3896 3897 port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port; 3898 3899 smac = (u8 *)hr_dev->dev_addr[port]; 3900 /* when dmac equals smac or loop_idc is 1, it should loopback */ 3901 if (ether_addr_equal_unaligned(dmac, smac) || 3902 hr_dev->loop_idc == 0x1) { 3903 roce_set_bit(context->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 1); 3904 roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 0); 3905 } 3906 3907 if (attr_mask & IB_QP_DEST_QPN) { 3908 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M, 3909 V2_QPC_BYTE_56_DQPN_S, attr->dest_qp_num); 3910 roce_set_field(qpc_mask->byte_56_dqpn_err, 3911 V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0); 3912 } 3913 3914 /* Configure GID index */ 3915 port_num = rdma_ah_get_port_num(&attr->ah_attr); 3916 roce_set_field(context->byte_20_smac_sgid_idx, 3917 V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 3918 hns_get_gid_index(hr_dev, port_num - 1, 3919 grh->sgid_index)); 3920 roce_set_field(qpc_mask->byte_20_smac_sgid_idx, 3921 V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 0); 3922 memcpy(&(context->dmac), dmac, sizeof(u32)); 3923 roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M, 3924 V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&dmac[4]))); 3925 qpc_mask->dmac = 0; 3926 roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M, 3927 V2_QPC_BYTE_52_DMAC_S, 0); 3928 3929 /* mtu*(2^LP_PKTN_INI) should not bigger than 1 message length 64kb */ 3930 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M, 3931 V2_QPC_BYTE_56_LP_PKTN_INI_S, 0); 3932 roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M, 3933 V2_QPC_BYTE_56_LP_PKTN_INI_S, 0); 3934 3935 if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD) 3936 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M, 3937 V2_QPC_BYTE_24_MTU_S, IB_MTU_4096); 3938 else if (attr_mask & IB_QP_PATH_MTU) 3939 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M, 3940 V2_QPC_BYTE_24_MTU_S, attr->path_mtu); 3941 3942 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M, 3943 V2_QPC_BYTE_24_MTU_S, 0); 3944 3945 roce_set_field(context->byte_84_rq_ci_pi, 3946 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, 3947 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head); 3948 roce_set_field(qpc_mask->byte_84_rq_ci_pi, 3949 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, 3950 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0); 3951 3952 roce_set_field(qpc_mask->byte_84_rq_ci_pi, 3953 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M, 3954 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0); 3955 roce_set_bit(qpc_mask->byte_108_rx_reqepsn, 3956 V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0); 3957 roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M, 3958 V2_QPC_BYTE_96_RX_REQ_MSN_S, 0); 3959 roce_set_field(qpc_mask->byte_108_rx_reqepsn, 3960 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M, 3961 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0); 3962 3963 context->rq_rnr_timer = 0; 3964 qpc_mask->rq_rnr_timer = 0; 3965 3966 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M, 3967 V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0); 3968 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M, 3969 V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0); 3970 3971 /* rocee send 2^lp_sgen_ini segs every time */ 3972 roce_set_field(context->byte_168_irrl_idx, 3973 V2_QPC_BYTE_168_LP_SGEN_INI_M, 3974 V2_QPC_BYTE_168_LP_SGEN_INI_S, 3); 3975 roce_set_field(qpc_mask->byte_168_irrl_idx, 3976 V2_QPC_BYTE_168_LP_SGEN_INI_M, 3977 V2_QPC_BYTE_168_LP_SGEN_INI_S, 0); 3978 3979 return 0; 3980 } 3981 3982 static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, 3983 const struct ib_qp_attr *attr, int attr_mask, 3984 struct hns_roce_v2_qp_context *context, 3985 struct hns_roce_v2_qp_context *qpc_mask) 3986 { 3987 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 3988 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 3989 struct ib_device *ibdev = &hr_dev->ib_dev; 3990 u64 sge_cur_blk = 0; 3991 u64 sq_cur_blk = 0; 3992 u32 page_size; 3993 int count; 3994 3995 /* Search qp buf's mtts */ 3996 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL); 3997 if (count < 1) { 3998 ibdev_err(ibdev, "failed to find buf pa of QP(0x%lx)\n", 3999 hr_qp->qpn); 4000 return -EINVAL; 4001 } 4002 4003 if (hr_qp->sge.offset) { 4004 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); 4005 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 4006 hr_qp->sge.offset / page_size, 4007 &sge_cur_blk, 1, NULL); 4008 if (count < 1) { 4009 ibdev_err(ibdev, "failed to find sge pa of QP(0x%lx)\n", 4010 hr_qp->qpn); 4011 return -EINVAL; 4012 } 4013 } 4014 4015 /* Not support alternate path and path migration */ 4016 if (attr_mask & (IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE)) { 4017 ibdev_err(ibdev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask); 4018 return -EINVAL; 4019 } 4020 4021 /* 4022 * In v2 engine, software pass context and context mask to hardware 4023 * when modifying qp. If software need modify some fields in context, 4024 * we should set all bits of the relevant fields in context mask to 4025 * 0 at the same time, else set them to 0x1. 4026 */ 4027 context->sq_cur_blk_addr = cpu_to_le32(sq_cur_blk >> PAGE_ADDR_SHIFT); 4028 roce_set_field(context->byte_168_irrl_idx, 4029 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M, 4030 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 4031 sq_cur_blk >> (32 + PAGE_ADDR_SHIFT)); 4032 qpc_mask->sq_cur_blk_addr = 0; 4033 roce_set_field(qpc_mask->byte_168_irrl_idx, 4034 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M, 4035 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0); 4036 4037 context->sq_cur_sge_blk_addr = ((ibqp->qp_type == IB_QPT_GSI) || 4038 hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ? 4039 cpu_to_le32(sge_cur_blk >> 4040 PAGE_ADDR_SHIFT) : 0; 4041 roce_set_field(context->byte_184_irrl_idx, 4042 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M, 4043 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 4044 ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 4045 HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ? 4046 (sge_cur_blk >> 4047 (32 + PAGE_ADDR_SHIFT)) : 0); 4048 qpc_mask->sq_cur_sge_blk_addr = 0; 4049 roce_set_field(qpc_mask->byte_184_irrl_idx, 4050 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M, 4051 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0); 4052 4053 context->rx_sq_cur_blk_addr = 4054 cpu_to_le32(sq_cur_blk >> PAGE_ADDR_SHIFT); 4055 roce_set_field(context->byte_232_irrl_sge, 4056 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M, 4057 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 4058 sq_cur_blk >> (32 + PAGE_ADDR_SHIFT)); 4059 qpc_mask->rx_sq_cur_blk_addr = 0; 4060 roce_set_field(qpc_mask->byte_232_irrl_sge, 4061 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M, 4062 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 0); 4063 4064 /* 4065 * Set some fields in context to zero, Because the default values 4066 * of all fields in context are zero, we need not set them to 0 again. 4067 * but we should set the relevant fields of context mask to 0. 4068 */ 4069 roce_set_field(qpc_mask->byte_232_irrl_sge, 4070 V2_QPC_BYTE_232_IRRL_SGE_IDX_M, 4071 V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0); 4072 4073 roce_set_field(qpc_mask->byte_240_irrl_tail, 4074 V2_QPC_BYTE_240_RX_ACK_MSN_M, 4075 V2_QPC_BYTE_240_RX_ACK_MSN_S, 0); 4076 4077 roce_set_field(qpc_mask->byte_248_ack_psn, 4078 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M, 4079 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0); 4080 roce_set_bit(qpc_mask->byte_248_ack_psn, 4081 V2_QPC_BYTE_248_IRRL_PSN_VLD_S, 0); 4082 roce_set_field(qpc_mask->byte_248_ack_psn, 4083 V2_QPC_BYTE_248_IRRL_PSN_M, 4084 V2_QPC_BYTE_248_IRRL_PSN_S, 0); 4085 4086 roce_set_field(qpc_mask->byte_240_irrl_tail, 4087 V2_QPC_BYTE_240_IRRL_TAIL_REAL_M, 4088 V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0); 4089 4090 roce_set_field(qpc_mask->byte_220_retry_psn_msn, 4091 V2_QPC_BYTE_220_RETRY_MSG_MSN_M, 4092 V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0); 4093 4094 roce_set_bit(qpc_mask->byte_248_ack_psn, 4095 V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0); 4096 4097 roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M, 4098 V2_QPC_BYTE_212_CHECK_FLG_S, 0); 4099 4100 roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_LSN_M, 4101 V2_QPC_BYTE_212_LSN_S, 0x100); 4102 roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_LSN_M, 4103 V2_QPC_BYTE_212_LSN_S, 0); 4104 4105 roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M, 4106 V2_QPC_BYTE_196_IRRL_HEAD_S, 0); 4107 4108 return 0; 4109 } 4110 4111 static inline bool hns_roce_v2_check_qp_stat(enum ib_qp_state cur_state, 4112 enum ib_qp_state new_state) 4113 { 4114 4115 if ((cur_state != IB_QPS_RESET && 4116 (new_state == IB_QPS_ERR || new_state == IB_QPS_RESET)) || 4117 ((cur_state == IB_QPS_RTS || cur_state == IB_QPS_SQD) && 4118 (new_state == IB_QPS_RTS || new_state == IB_QPS_SQD)) || 4119 (cur_state == IB_QPS_SQE && new_state == IB_QPS_RTS)) 4120 return true; 4121 4122 return false; 4123 4124 } 4125 4126 static int hns_roce_v2_set_path(struct ib_qp *ibqp, 4127 const struct ib_qp_attr *attr, 4128 int attr_mask, 4129 struct hns_roce_v2_qp_context *context, 4130 struct hns_roce_v2_qp_context *qpc_mask) 4131 { 4132 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 4133 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4134 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4135 struct ib_device *ibdev = &hr_dev->ib_dev; 4136 const struct ib_gid_attr *gid_attr = NULL; 4137 int is_roce_protocol; 4138 u16 vlan_id = 0xffff; 4139 bool is_udp = false; 4140 u8 ib_port; 4141 u8 hr_port; 4142 int ret; 4143 4144 ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num : hr_qp->port + 1; 4145 hr_port = ib_port - 1; 4146 is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) && 4147 rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH; 4148 4149 if (is_roce_protocol) { 4150 gid_attr = attr->ah_attr.grh.sgid_attr; 4151 ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL); 4152 if (ret) 4153 return ret; 4154 4155 if (gid_attr) 4156 is_udp = (gid_attr->gid_type == 4157 IB_GID_TYPE_ROCE_UDP_ENCAP); 4158 } 4159 4160 if (vlan_id < VLAN_N_VID) { 4161 roce_set_bit(context->byte_76_srqn_op_en, 4162 V2_QPC_BYTE_76_RQ_VLAN_EN_S, 1); 4163 roce_set_bit(qpc_mask->byte_76_srqn_op_en, 4164 V2_QPC_BYTE_76_RQ_VLAN_EN_S, 0); 4165 roce_set_bit(context->byte_168_irrl_idx, 4166 V2_QPC_BYTE_168_SQ_VLAN_EN_S, 1); 4167 roce_set_bit(qpc_mask->byte_168_irrl_idx, 4168 V2_QPC_BYTE_168_SQ_VLAN_EN_S, 0); 4169 } 4170 4171 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M, 4172 V2_QPC_BYTE_24_VLAN_ID_S, vlan_id); 4173 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M, 4174 V2_QPC_BYTE_24_VLAN_ID_S, 0); 4175 4176 if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) { 4177 ibdev_err(ibdev, "sgid_index(%u) too large. max is %d\n", 4178 grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]); 4179 return -EINVAL; 4180 } 4181 4182 if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) { 4183 ibdev_err(ibdev, "ah attr is not RDMA roce type\n"); 4184 return -EINVAL; 4185 } 4186 4187 roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M, 4188 V2_QPC_BYTE_52_UDPSPN_S, 4189 is_udp ? 0x12b7 : 0); 4190 4191 roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M, 4192 V2_QPC_BYTE_52_UDPSPN_S, 0); 4193 4194 roce_set_field(context->byte_20_smac_sgid_idx, 4195 V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 4196 grh->sgid_index); 4197 4198 roce_set_field(qpc_mask->byte_20_smac_sgid_idx, 4199 V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 0); 4200 4201 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M, 4202 V2_QPC_BYTE_24_HOP_LIMIT_S, grh->hop_limit); 4203 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M, 4204 V2_QPC_BYTE_24_HOP_LIMIT_S, 0); 4205 4206 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B && is_udp) 4207 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M, 4208 V2_QPC_BYTE_24_TC_S, grh->traffic_class >> 2); 4209 else 4210 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M, 4211 V2_QPC_BYTE_24_TC_S, grh->traffic_class); 4212 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M, 4213 V2_QPC_BYTE_24_TC_S, 0); 4214 roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M, 4215 V2_QPC_BYTE_28_FL_S, grh->flow_label); 4216 roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_FL_M, 4217 V2_QPC_BYTE_28_FL_S, 0); 4218 memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw)); 4219 memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw)); 4220 roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M, 4221 V2_QPC_BYTE_28_SL_S, rdma_ah_get_sl(&attr->ah_attr)); 4222 roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M, 4223 V2_QPC_BYTE_28_SL_S, 0); 4224 hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); 4225 4226 return 0; 4227 } 4228 4229 static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp, 4230 const struct ib_qp_attr *attr, 4231 int attr_mask, 4232 enum ib_qp_state cur_state, 4233 enum ib_qp_state new_state, 4234 struct hns_roce_v2_qp_context *context, 4235 struct hns_roce_v2_qp_context *qpc_mask) 4236 { 4237 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4238 int ret = 0; 4239 4240 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { 4241 memset(qpc_mask, 0, sizeof(*qpc_mask)); 4242 modify_qp_reset_to_init(ibqp, attr, attr_mask, context, 4243 qpc_mask); 4244 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) { 4245 modify_qp_init_to_init(ibqp, attr, attr_mask, context, 4246 qpc_mask); 4247 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) { 4248 ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context, 4249 qpc_mask); 4250 if (ret) 4251 goto out; 4252 } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) { 4253 ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context, 4254 qpc_mask); 4255 if (ret) 4256 goto out; 4257 } else if (hns_roce_v2_check_qp_stat(cur_state, new_state)) { 4258 /* Nothing */ 4259 ; 4260 } else { 4261 ibdev_err(&hr_dev->ib_dev, "Illegal state for QP!\n"); 4262 ret = -EINVAL; 4263 goto out; 4264 } 4265 4266 out: 4267 return ret; 4268 } 4269 4270 static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp, 4271 const struct ib_qp_attr *attr, 4272 int attr_mask, 4273 struct hns_roce_v2_qp_context *context, 4274 struct hns_roce_v2_qp_context *qpc_mask) 4275 { 4276 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4277 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4278 int ret = 0; 4279 4280 if (attr_mask & IB_QP_AV) { 4281 ret = hns_roce_v2_set_path(ibqp, attr, attr_mask, context, 4282 qpc_mask); 4283 if (ret) 4284 return ret; 4285 } 4286 4287 if (attr_mask & IB_QP_TIMEOUT) { 4288 if (attr->timeout < 31) { 4289 roce_set_field(context->byte_28_at_fl, 4290 V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S, 4291 attr->timeout); 4292 roce_set_field(qpc_mask->byte_28_at_fl, 4293 V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S, 4294 0); 4295 } else { 4296 ibdev_warn(&hr_dev->ib_dev, 4297 "Local ACK timeout shall be 0 to 30.\n"); 4298 } 4299 } 4300 4301 if (attr_mask & IB_QP_RETRY_CNT) { 4302 roce_set_field(context->byte_212_lsn, 4303 V2_QPC_BYTE_212_RETRY_NUM_INIT_M, 4304 V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 4305 attr->retry_cnt); 4306 roce_set_field(qpc_mask->byte_212_lsn, 4307 V2_QPC_BYTE_212_RETRY_NUM_INIT_M, 4308 V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 0); 4309 4310 roce_set_field(context->byte_212_lsn, 4311 V2_QPC_BYTE_212_RETRY_CNT_M, 4312 V2_QPC_BYTE_212_RETRY_CNT_S, attr->retry_cnt); 4313 roce_set_field(qpc_mask->byte_212_lsn, 4314 V2_QPC_BYTE_212_RETRY_CNT_M, 4315 V2_QPC_BYTE_212_RETRY_CNT_S, 0); 4316 } 4317 4318 if (attr_mask & IB_QP_RNR_RETRY) { 4319 roce_set_field(context->byte_244_rnr_rxack, 4320 V2_QPC_BYTE_244_RNR_NUM_INIT_M, 4321 V2_QPC_BYTE_244_RNR_NUM_INIT_S, attr->rnr_retry); 4322 roce_set_field(qpc_mask->byte_244_rnr_rxack, 4323 V2_QPC_BYTE_244_RNR_NUM_INIT_M, 4324 V2_QPC_BYTE_244_RNR_NUM_INIT_S, 0); 4325 4326 roce_set_field(context->byte_244_rnr_rxack, 4327 V2_QPC_BYTE_244_RNR_CNT_M, 4328 V2_QPC_BYTE_244_RNR_CNT_S, attr->rnr_retry); 4329 roce_set_field(qpc_mask->byte_244_rnr_rxack, 4330 V2_QPC_BYTE_244_RNR_CNT_M, 4331 V2_QPC_BYTE_244_RNR_CNT_S, 0); 4332 } 4333 4334 /* RC&UC&UD required attr */ 4335 if (attr_mask & IB_QP_SQ_PSN) { 4336 roce_set_field(context->byte_172_sq_psn, 4337 V2_QPC_BYTE_172_SQ_CUR_PSN_M, 4338 V2_QPC_BYTE_172_SQ_CUR_PSN_S, attr->sq_psn); 4339 roce_set_field(qpc_mask->byte_172_sq_psn, 4340 V2_QPC_BYTE_172_SQ_CUR_PSN_M, 4341 V2_QPC_BYTE_172_SQ_CUR_PSN_S, 0); 4342 4343 roce_set_field(context->byte_196_sq_psn, 4344 V2_QPC_BYTE_196_SQ_MAX_PSN_M, 4345 V2_QPC_BYTE_196_SQ_MAX_PSN_S, attr->sq_psn); 4346 roce_set_field(qpc_mask->byte_196_sq_psn, 4347 V2_QPC_BYTE_196_SQ_MAX_PSN_M, 4348 V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0); 4349 4350 roce_set_field(context->byte_220_retry_psn_msn, 4351 V2_QPC_BYTE_220_RETRY_MSG_PSN_M, 4352 V2_QPC_BYTE_220_RETRY_MSG_PSN_S, attr->sq_psn); 4353 roce_set_field(qpc_mask->byte_220_retry_psn_msn, 4354 V2_QPC_BYTE_220_RETRY_MSG_PSN_M, 4355 V2_QPC_BYTE_220_RETRY_MSG_PSN_S, 0); 4356 4357 roce_set_field(context->byte_224_retry_msg, 4358 V2_QPC_BYTE_224_RETRY_MSG_PSN_M, 4359 V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 4360 attr->sq_psn >> V2_QPC_BYTE_220_RETRY_MSG_PSN_S); 4361 roce_set_field(qpc_mask->byte_224_retry_msg, 4362 V2_QPC_BYTE_224_RETRY_MSG_PSN_M, 4363 V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0); 4364 4365 roce_set_field(context->byte_224_retry_msg, 4366 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M, 4367 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 4368 attr->sq_psn); 4369 roce_set_field(qpc_mask->byte_224_retry_msg, 4370 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M, 4371 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 0); 4372 4373 roce_set_field(context->byte_244_rnr_rxack, 4374 V2_QPC_BYTE_244_RX_ACK_EPSN_M, 4375 V2_QPC_BYTE_244_RX_ACK_EPSN_S, attr->sq_psn); 4376 roce_set_field(qpc_mask->byte_244_rnr_rxack, 4377 V2_QPC_BYTE_244_RX_ACK_EPSN_M, 4378 V2_QPC_BYTE_244_RX_ACK_EPSN_S, 0); 4379 } 4380 4381 if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) && 4382 attr->max_dest_rd_atomic) { 4383 roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M, 4384 V2_QPC_BYTE_140_RR_MAX_S, 4385 fls(attr->max_dest_rd_atomic - 1)); 4386 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M, 4387 V2_QPC_BYTE_140_RR_MAX_S, 0); 4388 } 4389 4390 if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) { 4391 roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M, 4392 V2_QPC_BYTE_208_SR_MAX_S, 4393 fls(attr->max_rd_atomic - 1)); 4394 roce_set_field(qpc_mask->byte_208_irrl, 4395 V2_QPC_BYTE_208_SR_MAX_M, 4396 V2_QPC_BYTE_208_SR_MAX_S, 0); 4397 } 4398 4399 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) 4400 set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask); 4401 4402 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 4403 roce_set_field(context->byte_80_rnr_rx_cqn, 4404 V2_QPC_BYTE_80_MIN_RNR_TIME_M, 4405 V2_QPC_BYTE_80_MIN_RNR_TIME_S, 4406 attr->min_rnr_timer); 4407 roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, 4408 V2_QPC_BYTE_80_MIN_RNR_TIME_M, 4409 V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0); 4410 } 4411 4412 /* RC&UC required attr */ 4413 if (attr_mask & IB_QP_RQ_PSN) { 4414 roce_set_field(context->byte_108_rx_reqepsn, 4415 V2_QPC_BYTE_108_RX_REQ_EPSN_M, 4416 V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn); 4417 roce_set_field(qpc_mask->byte_108_rx_reqepsn, 4418 V2_QPC_BYTE_108_RX_REQ_EPSN_M, 4419 V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0); 4420 4421 roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M, 4422 V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1); 4423 roce_set_field(qpc_mask->byte_152_raq, 4424 V2_QPC_BYTE_152_RAQ_PSN_M, 4425 V2_QPC_BYTE_152_RAQ_PSN_S, 0); 4426 } 4427 4428 if (attr_mask & IB_QP_QKEY) { 4429 context->qkey_xrcd = cpu_to_le32(attr->qkey); 4430 qpc_mask->qkey_xrcd = 0; 4431 hr_qp->qkey = attr->qkey; 4432 } 4433 4434 return ret; 4435 } 4436 4437 static void hns_roce_v2_record_opt_fields(struct ib_qp *ibqp, 4438 const struct ib_qp_attr *attr, 4439 int attr_mask) 4440 { 4441 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4442 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4443 4444 if (attr_mask & IB_QP_ACCESS_FLAGS) 4445 hr_qp->atomic_rd_en = attr->qp_access_flags; 4446 4447 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) 4448 hr_qp->resp_depth = attr->max_dest_rd_atomic; 4449 if (attr_mask & IB_QP_PORT) { 4450 hr_qp->port = attr->port_num - 1; 4451 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; 4452 } 4453 } 4454 4455 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, 4456 const struct ib_qp_attr *attr, 4457 int attr_mask, enum ib_qp_state cur_state, 4458 enum ib_qp_state new_state) 4459 { 4460 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4461 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4462 struct hns_roce_v2_qp_context ctx[2]; 4463 struct hns_roce_v2_qp_context *context = ctx; 4464 struct hns_roce_v2_qp_context *qpc_mask = ctx + 1; 4465 struct ib_device *ibdev = &hr_dev->ib_dev; 4466 unsigned long sq_flag = 0; 4467 unsigned long rq_flag = 0; 4468 int ret; 4469 4470 /* 4471 * In v2 engine, software pass context and context mask to hardware 4472 * when modifying qp. If software need modify some fields in context, 4473 * we should set all bits of the relevant fields in context mask to 4474 * 0 at the same time, else set them to 0x1. 4475 */ 4476 memset(context, 0, sizeof(*context)); 4477 memset(qpc_mask, 0xff, sizeof(*qpc_mask)); 4478 ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state, 4479 new_state, context, qpc_mask); 4480 if (ret) 4481 goto out; 4482 4483 /* When QP state is err, SQ and RQ WQE should be flushed */ 4484 if (new_state == IB_QPS_ERR) { 4485 spin_lock_irqsave(&hr_qp->sq.lock, sq_flag); 4486 hr_qp->state = IB_QPS_ERR; 4487 roce_set_field(context->byte_160_sq_ci_pi, 4488 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M, 4489 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 4490 hr_qp->sq.head); 4491 roce_set_field(qpc_mask->byte_160_sq_ci_pi, 4492 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M, 4493 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0); 4494 spin_unlock_irqrestore(&hr_qp->sq.lock, sq_flag); 4495 4496 if (!ibqp->srq) { 4497 spin_lock_irqsave(&hr_qp->rq.lock, rq_flag); 4498 roce_set_field(context->byte_84_rq_ci_pi, 4499 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, 4500 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 4501 hr_qp->rq.head); 4502 roce_set_field(qpc_mask->byte_84_rq_ci_pi, 4503 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, 4504 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0); 4505 spin_unlock_irqrestore(&hr_qp->rq.lock, rq_flag); 4506 } 4507 } 4508 4509 /* Configure the optional fields */ 4510 ret = hns_roce_v2_set_opt_fields(ibqp, attr, attr_mask, context, 4511 qpc_mask); 4512 if (ret) 4513 goto out; 4514 4515 roce_set_bit(context->byte_108_rx_reqepsn, V2_QPC_BYTE_108_INV_CREDIT_S, 4516 ibqp->srq ? 1 : 0); 4517 roce_set_bit(qpc_mask->byte_108_rx_reqepsn, 4518 V2_QPC_BYTE_108_INV_CREDIT_S, 0); 4519 4520 /* Every status migrate must change state */ 4521 roce_set_field(context->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M, 4522 V2_QPC_BYTE_60_QP_ST_S, new_state); 4523 roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M, 4524 V2_QPC_BYTE_60_QP_ST_S, 0); 4525 4526 /* SW pass context to HW */ 4527 ret = hns_roce_v2_qp_modify(hr_dev, ctx, hr_qp); 4528 if (ret) { 4529 ibdev_err(ibdev, "failed to modify QP, ret = %d\n", ret); 4530 goto out; 4531 } 4532 4533 hr_qp->state = new_state; 4534 4535 hns_roce_v2_record_opt_fields(ibqp, attr, attr_mask); 4536 4537 if (new_state == IB_QPS_RESET && !ibqp->uobject) { 4538 hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn, 4539 ibqp->srq ? to_hr_srq(ibqp->srq) : NULL); 4540 if (ibqp->send_cq != ibqp->recv_cq) 4541 hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq), 4542 hr_qp->qpn, NULL); 4543 4544 hr_qp->rq.head = 0; 4545 hr_qp->rq.tail = 0; 4546 hr_qp->sq.head = 0; 4547 hr_qp->sq.tail = 0; 4548 hr_qp->next_sge = 0; 4549 if (hr_qp->rq.wqe_cnt) 4550 *hr_qp->rdb.db_record = 0; 4551 } 4552 4553 out: 4554 return ret; 4555 } 4556 4557 static inline enum ib_qp_state to_ib_qp_st(enum hns_roce_v2_qp_state state) 4558 { 4559 switch (state) { 4560 case HNS_ROCE_QP_ST_RST: return IB_QPS_RESET; 4561 case HNS_ROCE_QP_ST_INIT: return IB_QPS_INIT; 4562 case HNS_ROCE_QP_ST_RTR: return IB_QPS_RTR; 4563 case HNS_ROCE_QP_ST_RTS: return IB_QPS_RTS; 4564 case HNS_ROCE_QP_ST_SQ_DRAINING: 4565 case HNS_ROCE_QP_ST_SQD: return IB_QPS_SQD; 4566 case HNS_ROCE_QP_ST_SQER: return IB_QPS_SQE; 4567 case HNS_ROCE_QP_ST_ERR: return IB_QPS_ERR; 4568 default: return -1; 4569 } 4570 } 4571 4572 static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, 4573 struct hns_roce_qp *hr_qp, 4574 struct hns_roce_v2_qp_context *hr_context) 4575 { 4576 struct hns_roce_cmd_mailbox *mailbox; 4577 int ret; 4578 4579 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 4580 if (IS_ERR(mailbox)) 4581 return PTR_ERR(mailbox); 4582 4583 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0, 4584 HNS_ROCE_CMD_QUERY_QPC, 4585 HNS_ROCE_CMD_TIMEOUT_MSECS); 4586 if (ret) 4587 goto out; 4588 4589 memcpy(hr_context, mailbox->buf, sizeof(*hr_context)); 4590 4591 out: 4592 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 4593 return ret; 4594 } 4595 4596 static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 4597 int qp_attr_mask, 4598 struct ib_qp_init_attr *qp_init_attr) 4599 { 4600 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4601 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4602 struct hns_roce_v2_qp_context context = {}; 4603 struct ib_device *ibdev = &hr_dev->ib_dev; 4604 int tmp_qp_state; 4605 int state; 4606 int ret; 4607 4608 memset(qp_attr, 0, sizeof(*qp_attr)); 4609 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 4610 4611 mutex_lock(&hr_qp->mutex); 4612 4613 if (hr_qp->state == IB_QPS_RESET) { 4614 qp_attr->qp_state = IB_QPS_RESET; 4615 ret = 0; 4616 goto done; 4617 } 4618 4619 ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, &context); 4620 if (ret) { 4621 ibdev_err(ibdev, "failed to query QPC, ret = %d\n", ret); 4622 ret = -EINVAL; 4623 goto out; 4624 } 4625 4626 state = roce_get_field(context.byte_60_qpst_tempid, 4627 V2_QPC_BYTE_60_QP_ST_M, V2_QPC_BYTE_60_QP_ST_S); 4628 tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state); 4629 if (tmp_qp_state == -1) { 4630 ibdev_err(ibdev, "Illegal ib_qp_state\n"); 4631 ret = -EINVAL; 4632 goto out; 4633 } 4634 hr_qp->state = (u8)tmp_qp_state; 4635 qp_attr->qp_state = (enum ib_qp_state)hr_qp->state; 4636 qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context.byte_24_mtu_tc, 4637 V2_QPC_BYTE_24_MTU_M, 4638 V2_QPC_BYTE_24_MTU_S); 4639 qp_attr->path_mig_state = IB_MIG_ARMED; 4640 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 4641 if (hr_qp->ibqp.qp_type == IB_QPT_UD) 4642 qp_attr->qkey = V2_QKEY_VAL; 4643 4644 qp_attr->rq_psn = roce_get_field(context.byte_108_rx_reqepsn, 4645 V2_QPC_BYTE_108_RX_REQ_EPSN_M, 4646 V2_QPC_BYTE_108_RX_REQ_EPSN_S); 4647 qp_attr->sq_psn = (u32)roce_get_field(context.byte_172_sq_psn, 4648 V2_QPC_BYTE_172_SQ_CUR_PSN_M, 4649 V2_QPC_BYTE_172_SQ_CUR_PSN_S); 4650 qp_attr->dest_qp_num = (u8)roce_get_field(context.byte_56_dqpn_err, 4651 V2_QPC_BYTE_56_DQPN_M, 4652 V2_QPC_BYTE_56_DQPN_S); 4653 qp_attr->qp_access_flags = ((roce_get_bit(context.byte_76_srqn_op_en, 4654 V2_QPC_BYTE_76_RRE_S)) << V2_QP_RRE_S) | 4655 ((roce_get_bit(context.byte_76_srqn_op_en, 4656 V2_QPC_BYTE_76_RWE_S)) << V2_QP_RWE_S) | 4657 ((roce_get_bit(context.byte_76_srqn_op_en, 4658 V2_QPC_BYTE_76_ATE_S)) << V2_QP_ATE_S); 4659 4660 if (hr_qp->ibqp.qp_type == IB_QPT_RC || 4661 hr_qp->ibqp.qp_type == IB_QPT_UC) { 4662 struct ib_global_route *grh = 4663 rdma_ah_retrieve_grh(&qp_attr->ah_attr); 4664 4665 rdma_ah_set_sl(&qp_attr->ah_attr, 4666 roce_get_field(context.byte_28_at_fl, 4667 V2_QPC_BYTE_28_SL_M, 4668 V2_QPC_BYTE_28_SL_S)); 4669 grh->flow_label = roce_get_field(context.byte_28_at_fl, 4670 V2_QPC_BYTE_28_FL_M, 4671 V2_QPC_BYTE_28_FL_S); 4672 grh->sgid_index = roce_get_field(context.byte_20_smac_sgid_idx, 4673 V2_QPC_BYTE_20_SGID_IDX_M, 4674 V2_QPC_BYTE_20_SGID_IDX_S); 4675 grh->hop_limit = roce_get_field(context.byte_24_mtu_tc, 4676 V2_QPC_BYTE_24_HOP_LIMIT_M, 4677 V2_QPC_BYTE_24_HOP_LIMIT_S); 4678 grh->traffic_class = roce_get_field(context.byte_24_mtu_tc, 4679 V2_QPC_BYTE_24_TC_M, 4680 V2_QPC_BYTE_24_TC_S); 4681 4682 memcpy(grh->dgid.raw, context.dgid, sizeof(grh->dgid.raw)); 4683 } 4684 4685 qp_attr->port_num = hr_qp->port + 1; 4686 qp_attr->sq_draining = 0; 4687 qp_attr->max_rd_atomic = 1 << roce_get_field(context.byte_208_irrl, 4688 V2_QPC_BYTE_208_SR_MAX_M, 4689 V2_QPC_BYTE_208_SR_MAX_S); 4690 qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context.byte_140_raq, 4691 V2_QPC_BYTE_140_RR_MAX_M, 4692 V2_QPC_BYTE_140_RR_MAX_S); 4693 qp_attr->min_rnr_timer = (u8)roce_get_field(context.byte_80_rnr_rx_cqn, 4694 V2_QPC_BYTE_80_MIN_RNR_TIME_M, 4695 V2_QPC_BYTE_80_MIN_RNR_TIME_S); 4696 qp_attr->timeout = (u8)roce_get_field(context.byte_28_at_fl, 4697 V2_QPC_BYTE_28_AT_M, 4698 V2_QPC_BYTE_28_AT_S); 4699 qp_attr->retry_cnt = roce_get_field(context.byte_212_lsn, 4700 V2_QPC_BYTE_212_RETRY_CNT_M, 4701 V2_QPC_BYTE_212_RETRY_CNT_S); 4702 qp_attr->rnr_retry = le32_to_cpu(context.rq_rnr_timer); 4703 4704 done: 4705 qp_attr->cur_qp_state = qp_attr->qp_state; 4706 qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt; 4707 qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs; 4708 4709 if (!ibqp->uobject) { 4710 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt; 4711 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs; 4712 } else { 4713 qp_attr->cap.max_send_wr = 0; 4714 qp_attr->cap.max_send_sge = 0; 4715 } 4716 4717 qp_init_attr->cap = qp_attr->cap; 4718 4719 out: 4720 mutex_unlock(&hr_qp->mutex); 4721 return ret; 4722 } 4723 4724 static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, 4725 struct hns_roce_qp *hr_qp, 4726 struct ib_udata *udata) 4727 { 4728 struct ib_device *ibdev = &hr_dev->ib_dev; 4729 struct hns_roce_cq *send_cq, *recv_cq; 4730 unsigned long flags; 4731 int ret = 0; 4732 4733 if (hr_qp->ibqp.qp_type == IB_QPT_RC && hr_qp->state != IB_QPS_RESET) { 4734 /* Modify qp to reset before destroying qp */ 4735 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0, 4736 hr_qp->state, IB_QPS_RESET); 4737 if (ret) 4738 ibdev_err(ibdev, 4739 "failed to modify QP to RST, ret = %d\n", 4740 ret); 4741 } 4742 4743 send_cq = hr_qp->ibqp.send_cq ? to_hr_cq(hr_qp->ibqp.send_cq) : NULL; 4744 recv_cq = hr_qp->ibqp.recv_cq ? to_hr_cq(hr_qp->ibqp.recv_cq) : NULL; 4745 4746 spin_lock_irqsave(&hr_dev->qp_list_lock, flags); 4747 hns_roce_lock_cqs(send_cq, recv_cq); 4748 4749 if (!udata) { 4750 if (recv_cq) 4751 __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn, 4752 (hr_qp->ibqp.srq ? 4753 to_hr_srq(hr_qp->ibqp.srq) : 4754 NULL)); 4755 4756 if (send_cq && send_cq != recv_cq) 4757 __hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL); 4758 4759 } 4760 4761 hns_roce_qp_remove(hr_dev, hr_qp); 4762 4763 hns_roce_unlock_cqs(send_cq, recv_cq); 4764 spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); 4765 4766 return ret; 4767 } 4768 4769 static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 4770 { 4771 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4772 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4773 int ret; 4774 4775 ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata); 4776 if (ret) 4777 ibdev_err(&hr_dev->ib_dev, 4778 "failed to destroy QP 0x%06lx, ret = %d\n", 4779 hr_qp->qpn, ret); 4780 4781 hns_roce_qp_destroy(hr_dev, hr_qp, udata); 4782 4783 return 0; 4784 } 4785 4786 static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev, 4787 struct hns_roce_qp *hr_qp) 4788 { 4789 struct ib_device *ibdev = &hr_dev->ib_dev; 4790 struct hns_roce_sccc_clr_done *resp; 4791 struct hns_roce_sccc_clr *clr; 4792 struct hns_roce_cmq_desc desc; 4793 int ret, i; 4794 4795 mutex_lock(&hr_dev->qp_table.scc_mutex); 4796 4797 /* set scc ctx clear done flag */ 4798 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCCC, false); 4799 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 4800 if (ret) { 4801 ibdev_err(ibdev, "failed to reset SCC ctx, ret = %d\n", ret); 4802 goto out; 4803 } 4804 4805 /* clear scc context */ 4806 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLR_SCCC, false); 4807 clr = (struct hns_roce_sccc_clr *)desc.data; 4808 clr->qpn = cpu_to_le32(hr_qp->qpn); 4809 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 4810 if (ret) { 4811 ibdev_err(ibdev, "failed to clear SCC ctx, ret = %d\n", ret); 4812 goto out; 4813 } 4814 4815 /* query scc context clear is done or not */ 4816 resp = (struct hns_roce_sccc_clr_done *)desc.data; 4817 for (i = 0; i <= HNS_ROCE_CMQ_SCC_CLR_DONE_CNT; i++) { 4818 hns_roce_cmq_setup_basic_desc(&desc, 4819 HNS_ROCE_OPC_QUERY_SCCC, true); 4820 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 4821 if (ret) { 4822 ibdev_err(ibdev, "failed to query clr cmq, ret = %d\n", 4823 ret); 4824 goto out; 4825 } 4826 4827 if (resp->clr_done) 4828 goto out; 4829 4830 msleep(20); 4831 } 4832 4833 ibdev_err(ibdev, "Query SCC clr done flag overtime.\n"); 4834 ret = -ETIMEDOUT; 4835 4836 out: 4837 mutex_unlock(&hr_dev->qp_table.scc_mutex); 4838 return ret; 4839 } 4840 4841 static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) 4842 { 4843 struct hns_roce_dev *hr_dev = to_hr_dev(cq->device); 4844 struct hns_roce_v2_cq_context *cq_context; 4845 struct hns_roce_cq *hr_cq = to_hr_cq(cq); 4846 struct hns_roce_v2_cq_context *cqc_mask; 4847 struct hns_roce_cmd_mailbox *mailbox; 4848 int ret; 4849 4850 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 4851 if (IS_ERR(mailbox)) 4852 return PTR_ERR(mailbox); 4853 4854 cq_context = mailbox->buf; 4855 cqc_mask = (struct hns_roce_v2_cq_context *)mailbox->buf + 1; 4856 4857 memset(cqc_mask, 0xff, sizeof(*cqc_mask)); 4858 4859 roce_set_field(cq_context->byte_56_cqe_period_maxcnt, 4860 V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S, 4861 cq_count); 4862 roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt, 4863 V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S, 4864 0); 4865 roce_set_field(cq_context->byte_56_cqe_period_maxcnt, 4866 V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S, 4867 cq_period); 4868 roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt, 4869 V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S, 4870 0); 4871 4872 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_cq->cqn, 1, 4873 HNS_ROCE_CMD_MODIFY_CQC, 4874 HNS_ROCE_CMD_TIMEOUT_MSECS); 4875 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 4876 if (ret) 4877 ibdev_err(&hr_dev->ib_dev, 4878 "failed to process cmd when modifying CQ, ret = %d\n", 4879 ret); 4880 4881 return ret; 4882 } 4883 4884 static void hns_roce_irq_work_handle(struct work_struct *work) 4885 { 4886 struct hns_roce_work *irq_work = 4887 container_of(work, struct hns_roce_work, work); 4888 struct ib_device *ibdev = &irq_work->hr_dev->ib_dev; 4889 u32 qpn = irq_work->qpn; 4890 u32 cqn = irq_work->cqn; 4891 4892 switch (irq_work->event_type) { 4893 case HNS_ROCE_EVENT_TYPE_PATH_MIG: 4894 ibdev_info(ibdev, "Path migrated succeeded.\n"); 4895 break; 4896 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: 4897 ibdev_warn(ibdev, "Path migration failed.\n"); 4898 break; 4899 case HNS_ROCE_EVENT_TYPE_COMM_EST: 4900 break; 4901 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: 4902 ibdev_warn(ibdev, "Send queue drained.\n"); 4903 break; 4904 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: 4905 ibdev_err(ibdev, "Local work queue 0x%x catast error, sub_event type is: %d\n", 4906 qpn, irq_work->sub_type); 4907 break; 4908 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: 4909 ibdev_err(ibdev, "Invalid request local work queue 0x%x error.\n", 4910 qpn); 4911 break; 4912 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: 4913 ibdev_err(ibdev, "Local access violation work queue 0x%x error, sub_event type is: %d\n", 4914 qpn, irq_work->sub_type); 4915 break; 4916 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: 4917 ibdev_warn(ibdev, "SRQ limit reach.\n"); 4918 break; 4919 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: 4920 ibdev_warn(ibdev, "SRQ last wqe reach.\n"); 4921 break; 4922 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: 4923 ibdev_err(ibdev, "SRQ catas error.\n"); 4924 break; 4925 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: 4926 ibdev_err(ibdev, "CQ 0x%x access err.\n", cqn); 4927 break; 4928 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: 4929 ibdev_warn(ibdev, "CQ 0x%x overflow\n", cqn); 4930 break; 4931 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW: 4932 ibdev_warn(ibdev, "DB overflow.\n"); 4933 break; 4934 case HNS_ROCE_EVENT_TYPE_FLR: 4935 ibdev_warn(ibdev, "Function level reset.\n"); 4936 break; 4937 default: 4938 break; 4939 } 4940 4941 kfree(irq_work); 4942 } 4943 4944 static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev, 4945 struct hns_roce_eq *eq, 4946 u32 qpn, u32 cqn) 4947 { 4948 struct hns_roce_work *irq_work; 4949 4950 irq_work = kzalloc(sizeof(struct hns_roce_work), GFP_ATOMIC); 4951 if (!irq_work) 4952 return; 4953 4954 INIT_WORK(&(irq_work->work), hns_roce_irq_work_handle); 4955 irq_work->hr_dev = hr_dev; 4956 irq_work->qpn = qpn; 4957 irq_work->cqn = cqn; 4958 irq_work->event_type = eq->event_type; 4959 irq_work->sub_type = eq->sub_type; 4960 queue_work(hr_dev->irq_workq, &(irq_work->work)); 4961 } 4962 4963 static void set_eq_cons_index_v2(struct hns_roce_eq *eq) 4964 { 4965 struct hns_roce_dev *hr_dev = eq->hr_dev; 4966 __le32 doorbell[2] = {}; 4967 4968 if (eq->type_flag == HNS_ROCE_AEQ) { 4969 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M, 4970 HNS_ROCE_V2_EQ_DB_CMD_S, 4971 eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ? 4972 HNS_ROCE_EQ_DB_CMD_AEQ : 4973 HNS_ROCE_EQ_DB_CMD_AEQ_ARMED); 4974 } else { 4975 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_TAG_M, 4976 HNS_ROCE_V2_EQ_DB_TAG_S, eq->eqn); 4977 4978 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M, 4979 HNS_ROCE_V2_EQ_DB_CMD_S, 4980 eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ? 4981 HNS_ROCE_EQ_DB_CMD_CEQ : 4982 HNS_ROCE_EQ_DB_CMD_CEQ_ARMED); 4983 } 4984 4985 roce_set_field(doorbell[1], HNS_ROCE_V2_EQ_DB_PARA_M, 4986 HNS_ROCE_V2_EQ_DB_PARA_S, 4987 (eq->cons_index & HNS_ROCE_V2_CONS_IDX_M)); 4988 4989 hns_roce_write64(hr_dev, doorbell, eq->doorbell); 4990 } 4991 4992 static inline void *get_eqe_buf(struct hns_roce_eq *eq, unsigned long offset) 4993 { 4994 u32 buf_chk_sz; 4995 4996 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT); 4997 if (eq->buf.nbufs == 1) 4998 return eq->buf.direct.buf + offset % buf_chk_sz; 4999 else 5000 return eq->buf.page_list[offset / buf_chk_sz].buf + 5001 offset % buf_chk_sz; 5002 } 5003 5004 static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq) 5005 { 5006 struct hns_roce_aeqe *aeqe; 5007 5008 aeqe = get_eqe_buf(eq, (eq->cons_index & (eq->entries - 1)) * 5009 HNS_ROCE_AEQ_ENTRY_SIZE); 5010 return (roce_get_bit(aeqe->asyn, HNS_ROCE_V2_AEQ_AEQE_OWNER_S) ^ 5011 !!(eq->cons_index & eq->entries)) ? aeqe : NULL; 5012 } 5013 5014 static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev, 5015 struct hns_roce_eq *eq) 5016 { 5017 struct device *dev = hr_dev->dev; 5018 struct hns_roce_aeqe *aeqe = next_aeqe_sw_v2(eq); 5019 int aeqe_found = 0; 5020 int event_type; 5021 int sub_type; 5022 u32 srqn; 5023 u32 qpn; 5024 u32 cqn; 5025 5026 while (aeqe) { 5027 /* Make sure we read AEQ entry after we have checked the 5028 * ownership bit 5029 */ 5030 dma_rmb(); 5031 5032 event_type = roce_get_field(aeqe->asyn, 5033 HNS_ROCE_V2_AEQE_EVENT_TYPE_M, 5034 HNS_ROCE_V2_AEQE_EVENT_TYPE_S); 5035 sub_type = roce_get_field(aeqe->asyn, 5036 HNS_ROCE_V2_AEQE_SUB_TYPE_M, 5037 HNS_ROCE_V2_AEQE_SUB_TYPE_S); 5038 qpn = roce_get_field(aeqe->event.qp_event.qp, 5039 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M, 5040 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S); 5041 cqn = roce_get_field(aeqe->event.cq_event.cq, 5042 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M, 5043 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S); 5044 srqn = roce_get_field(aeqe->event.srq_event.srq, 5045 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M, 5046 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S); 5047 5048 switch (event_type) { 5049 case HNS_ROCE_EVENT_TYPE_PATH_MIG: 5050 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: 5051 case HNS_ROCE_EVENT_TYPE_COMM_EST: 5052 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: 5053 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: 5054 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: 5055 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: 5056 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: 5057 hns_roce_qp_event(hr_dev, qpn, event_type); 5058 break; 5059 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: 5060 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: 5061 hns_roce_srq_event(hr_dev, srqn, event_type); 5062 break; 5063 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: 5064 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: 5065 hns_roce_cq_event(hr_dev, cqn, event_type); 5066 break; 5067 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW: 5068 break; 5069 case HNS_ROCE_EVENT_TYPE_MB: 5070 hns_roce_cmd_event(hr_dev, 5071 le16_to_cpu(aeqe->event.cmd.token), 5072 aeqe->event.cmd.status, 5073 le64_to_cpu(aeqe->event.cmd.out_param)); 5074 break; 5075 case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW: 5076 break; 5077 case HNS_ROCE_EVENT_TYPE_FLR: 5078 break; 5079 default: 5080 dev_err(dev, "Unhandled event %d on EQ %d at idx %u.\n", 5081 event_type, eq->eqn, eq->cons_index); 5082 break; 5083 } 5084 5085 eq->event_type = event_type; 5086 eq->sub_type = sub_type; 5087 ++eq->cons_index; 5088 aeqe_found = 1; 5089 5090 if (eq->cons_index > (2 * eq->entries - 1)) 5091 eq->cons_index = 0; 5092 5093 hns_roce_v2_init_irq_work(hr_dev, eq, qpn, cqn); 5094 5095 aeqe = next_aeqe_sw_v2(eq); 5096 } 5097 5098 set_eq_cons_index_v2(eq); 5099 return aeqe_found; 5100 } 5101 5102 static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq) 5103 { 5104 struct hns_roce_ceqe *ceqe; 5105 5106 ceqe = get_eqe_buf(eq, (eq->cons_index & (eq->entries - 1)) * 5107 HNS_ROCE_CEQ_ENTRY_SIZE); 5108 return (!!(roce_get_bit(ceqe->comp, HNS_ROCE_V2_CEQ_CEQE_OWNER_S))) ^ 5109 (!!(eq->cons_index & eq->entries)) ? ceqe : NULL; 5110 } 5111 5112 static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev, 5113 struct hns_roce_eq *eq) 5114 { 5115 struct hns_roce_ceqe *ceqe = next_ceqe_sw_v2(eq); 5116 int ceqe_found = 0; 5117 u32 cqn; 5118 5119 while (ceqe) { 5120 /* Make sure we read CEQ entry after we have checked the 5121 * ownership bit 5122 */ 5123 dma_rmb(); 5124 5125 cqn = roce_get_field(ceqe->comp, HNS_ROCE_V2_CEQE_COMP_CQN_M, 5126 HNS_ROCE_V2_CEQE_COMP_CQN_S); 5127 5128 hns_roce_cq_completion(hr_dev, cqn); 5129 5130 ++eq->cons_index; 5131 ceqe_found = 1; 5132 5133 if (eq->cons_index > (EQ_DEPTH_COEFF * eq->entries - 1)) 5134 eq->cons_index = 0; 5135 5136 ceqe = next_ceqe_sw_v2(eq); 5137 } 5138 5139 set_eq_cons_index_v2(eq); 5140 5141 return ceqe_found; 5142 } 5143 5144 static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr) 5145 { 5146 struct hns_roce_eq *eq = eq_ptr; 5147 struct hns_roce_dev *hr_dev = eq->hr_dev; 5148 int int_work = 0; 5149 5150 if (eq->type_flag == HNS_ROCE_CEQ) 5151 /* Completion event interrupt */ 5152 int_work = hns_roce_v2_ceq_int(hr_dev, eq); 5153 else 5154 /* Asychronous event interrupt */ 5155 int_work = hns_roce_v2_aeq_int(hr_dev, eq); 5156 5157 return IRQ_RETVAL(int_work); 5158 } 5159 5160 static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id) 5161 { 5162 struct hns_roce_dev *hr_dev = dev_id; 5163 struct device *dev = hr_dev->dev; 5164 int int_work = 0; 5165 u32 int_st; 5166 u32 int_en; 5167 5168 /* Abnormal interrupt */ 5169 int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG); 5170 int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG); 5171 5172 if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) { 5173 struct pci_dev *pdev = hr_dev->pci_dev; 5174 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); 5175 const struct hnae3_ae_ops *ops = ae_dev->ops; 5176 5177 dev_err(dev, "AEQ overflow!\n"); 5178 5179 int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S; 5180 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st); 5181 5182 /* Set reset level for reset_event() */ 5183 if (ops->set_default_reset_request) 5184 ops->set_default_reset_request(ae_dev, 5185 HNAE3_FUNC_RESET); 5186 if (ops->reset_event) 5187 ops->reset_event(pdev, NULL); 5188 5189 int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S; 5190 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); 5191 5192 int_work = 1; 5193 } else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S)) { 5194 dev_err(dev, "BUS ERR!\n"); 5195 5196 int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S; 5197 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st); 5198 5199 int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S; 5200 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); 5201 5202 int_work = 1; 5203 } else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S)) { 5204 dev_err(dev, "OTHER ERR!\n"); 5205 5206 int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S; 5207 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st); 5208 5209 int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S; 5210 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); 5211 5212 int_work = 1; 5213 } else 5214 dev_err(dev, "There is no abnormal irq found!\n"); 5215 5216 return IRQ_RETVAL(int_work); 5217 } 5218 5219 static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev, 5220 int eq_num, int enable_flag) 5221 { 5222 int i; 5223 5224 if (enable_flag == EQ_ENABLE) { 5225 for (i = 0; i < eq_num; i++) 5226 roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG + 5227 i * EQ_REG_OFFSET, 5228 HNS_ROCE_V2_VF_EVENT_INT_EN_M); 5229 5230 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, 5231 HNS_ROCE_V2_VF_ABN_INT_EN_M); 5232 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, 5233 HNS_ROCE_V2_VF_ABN_INT_CFG_M); 5234 } else { 5235 for (i = 0; i < eq_num; i++) 5236 roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG + 5237 i * EQ_REG_OFFSET, 5238 HNS_ROCE_V2_VF_EVENT_INT_EN_M & 0x0); 5239 5240 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, 5241 HNS_ROCE_V2_VF_ABN_INT_EN_M & 0x0); 5242 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, 5243 HNS_ROCE_V2_VF_ABN_INT_CFG_M & 0x0); 5244 } 5245 } 5246 5247 static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn) 5248 { 5249 struct device *dev = hr_dev->dev; 5250 int ret; 5251 5252 if (eqn < hr_dev->caps.num_comp_vectors) 5253 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M, 5254 0, HNS_ROCE_CMD_DESTROY_CEQC, 5255 HNS_ROCE_CMD_TIMEOUT_MSECS); 5256 else 5257 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M, 5258 0, HNS_ROCE_CMD_DESTROY_AEQC, 5259 HNS_ROCE_CMD_TIMEOUT_MSECS); 5260 if (ret) 5261 dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn); 5262 } 5263 5264 static void free_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) 5265 { 5266 if (!eq->hop_num || eq->hop_num == HNS_ROCE_HOP_NUM_0) 5267 hns_roce_mtr_cleanup(hr_dev, &eq->mtr); 5268 hns_roce_buf_free(hr_dev, eq->buf.size, &eq->buf); 5269 } 5270 5271 static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev, 5272 struct hns_roce_eq *eq, 5273 void *mb_buf) 5274 { 5275 struct hns_roce_eq_context *eqc; 5276 u64 ba[MTT_MIN_COUNT] = { 0 }; 5277 int count; 5278 5279 eqc = mb_buf; 5280 memset(eqc, 0, sizeof(struct hns_roce_eq_context)); 5281 5282 /* init eqc */ 5283 eq->doorbell = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG; 5284 eq->hop_num = hr_dev->caps.eqe_hop_num; 5285 eq->cons_index = 0; 5286 eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0; 5287 eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0; 5288 eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED; 5289 eq->eqe_ba_pg_sz = hr_dev->caps.eqe_ba_pg_sz; 5290 eq->eqe_buf_pg_sz = hr_dev->caps.eqe_buf_pg_sz; 5291 eq->shift = ilog2((unsigned int)eq->entries); 5292 5293 /* if not muti-hop, eqe buffer only use one trunk */ 5294 if (!eq->hop_num || eq->hop_num == HNS_ROCE_HOP_NUM_0) { 5295 eq->eqe_ba = eq->buf.direct.map; 5296 eq->cur_eqe_ba = eq->eqe_ba; 5297 if (eq->buf.npages > 1) 5298 eq->nxt_eqe_ba = eq->eqe_ba + (1 << eq->eqe_buf_pg_sz); 5299 else 5300 eq->nxt_eqe_ba = eq->eqe_ba; 5301 } else { 5302 count = hns_roce_mtr_find(hr_dev, &eq->mtr, 0, ba, 5303 MTT_MIN_COUNT, &eq->eqe_ba); 5304 eq->cur_eqe_ba = ba[0]; 5305 if (count > 1) 5306 eq->nxt_eqe_ba = ba[1]; 5307 else 5308 eq->nxt_eqe_ba = ba[0]; 5309 } 5310 5311 /* set eqc state */ 5312 roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQ_ST_M, HNS_ROCE_EQC_EQ_ST_S, 5313 HNS_ROCE_V2_EQ_STATE_VALID); 5314 5315 /* set eqe hop num */ 5316 roce_set_field(eqc->byte_4, HNS_ROCE_EQC_HOP_NUM_M, 5317 HNS_ROCE_EQC_HOP_NUM_S, eq->hop_num); 5318 5319 /* set eqc over_ignore */ 5320 roce_set_field(eqc->byte_4, HNS_ROCE_EQC_OVER_IGNORE_M, 5321 HNS_ROCE_EQC_OVER_IGNORE_S, eq->over_ignore); 5322 5323 /* set eqc coalesce */ 5324 roce_set_field(eqc->byte_4, HNS_ROCE_EQC_COALESCE_M, 5325 HNS_ROCE_EQC_COALESCE_S, eq->coalesce); 5326 5327 /* set eqc arm_state */ 5328 roce_set_field(eqc->byte_4, HNS_ROCE_EQC_ARM_ST_M, 5329 HNS_ROCE_EQC_ARM_ST_S, eq->arm_st); 5330 5331 /* set eqn */ 5332 roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQN_M, HNS_ROCE_EQC_EQN_S, 5333 eq->eqn); 5334 5335 /* set eqe_cnt */ 5336 roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQE_CNT_M, 5337 HNS_ROCE_EQC_EQE_CNT_S, HNS_ROCE_EQ_INIT_EQE_CNT); 5338 5339 /* set eqe_ba_pg_sz */ 5340 roce_set_field(eqc->byte_8, HNS_ROCE_EQC_BA_PG_SZ_M, 5341 HNS_ROCE_EQC_BA_PG_SZ_S, 5342 eq->eqe_ba_pg_sz + PG_SHIFT_OFFSET); 5343 5344 /* set eqe_buf_pg_sz */ 5345 roce_set_field(eqc->byte_8, HNS_ROCE_EQC_BUF_PG_SZ_M, 5346 HNS_ROCE_EQC_BUF_PG_SZ_S, 5347 eq->eqe_buf_pg_sz + PG_SHIFT_OFFSET); 5348 5349 /* set eq_producer_idx */ 5350 roce_set_field(eqc->byte_8, HNS_ROCE_EQC_PROD_INDX_M, 5351 HNS_ROCE_EQC_PROD_INDX_S, HNS_ROCE_EQ_INIT_PROD_IDX); 5352 5353 /* set eq_max_cnt */ 5354 roce_set_field(eqc->byte_12, HNS_ROCE_EQC_MAX_CNT_M, 5355 HNS_ROCE_EQC_MAX_CNT_S, eq->eq_max_cnt); 5356 5357 /* set eq_period */ 5358 roce_set_field(eqc->byte_12, HNS_ROCE_EQC_PERIOD_M, 5359 HNS_ROCE_EQC_PERIOD_S, eq->eq_period); 5360 5361 /* set eqe_report_timer */ 5362 roce_set_field(eqc->eqe_report_timer, HNS_ROCE_EQC_REPORT_TIMER_M, 5363 HNS_ROCE_EQC_REPORT_TIMER_S, 5364 HNS_ROCE_EQ_INIT_REPORT_TIMER); 5365 5366 /* set eqe_ba [34:3] */ 5367 roce_set_field(eqc->eqe_ba0, HNS_ROCE_EQC_EQE_BA_L_M, 5368 HNS_ROCE_EQC_EQE_BA_L_S, eq->eqe_ba >> 3); 5369 5370 /* set eqe_ba [64:35] */ 5371 roce_set_field(eqc->eqe_ba1, HNS_ROCE_EQC_EQE_BA_H_M, 5372 HNS_ROCE_EQC_EQE_BA_H_S, eq->eqe_ba >> 35); 5373 5374 /* set eq shift */ 5375 roce_set_field(eqc->byte_28, HNS_ROCE_EQC_SHIFT_M, HNS_ROCE_EQC_SHIFT_S, 5376 eq->shift); 5377 5378 /* set eq MSI_IDX */ 5379 roce_set_field(eqc->byte_28, HNS_ROCE_EQC_MSI_INDX_M, 5380 HNS_ROCE_EQC_MSI_INDX_S, HNS_ROCE_EQ_INIT_MSI_IDX); 5381 5382 /* set cur_eqe_ba [27:12] */ 5383 roce_set_field(eqc->byte_28, HNS_ROCE_EQC_CUR_EQE_BA_L_M, 5384 HNS_ROCE_EQC_CUR_EQE_BA_L_S, eq->cur_eqe_ba >> 12); 5385 5386 /* set cur_eqe_ba [59:28] */ 5387 roce_set_field(eqc->byte_32, HNS_ROCE_EQC_CUR_EQE_BA_M_M, 5388 HNS_ROCE_EQC_CUR_EQE_BA_M_S, eq->cur_eqe_ba >> 28); 5389 5390 /* set cur_eqe_ba [63:60] */ 5391 roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CUR_EQE_BA_H_M, 5392 HNS_ROCE_EQC_CUR_EQE_BA_H_S, eq->cur_eqe_ba >> 60); 5393 5394 /* set eq consumer idx */ 5395 roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CONS_INDX_M, 5396 HNS_ROCE_EQC_CONS_INDX_S, HNS_ROCE_EQ_INIT_CONS_IDX); 5397 5398 /* set nex_eqe_ba[43:12] */ 5399 roce_set_field(eqc->nxt_eqe_ba0, HNS_ROCE_EQC_NXT_EQE_BA_L_M, 5400 HNS_ROCE_EQC_NXT_EQE_BA_L_S, eq->nxt_eqe_ba >> 12); 5401 5402 /* set nex_eqe_ba[63:44] */ 5403 roce_set_field(eqc->nxt_eqe_ba1, HNS_ROCE_EQC_NXT_EQE_BA_H_M, 5404 HNS_ROCE_EQC_NXT_EQE_BA_H_S, eq->nxt_eqe_ba >> 44); 5405 } 5406 5407 static int map_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq, 5408 u32 page_shift) 5409 { 5410 struct hns_roce_buf_region region = {}; 5411 dma_addr_t *buf_list = NULL; 5412 int ba_num; 5413 int ret; 5414 5415 ba_num = DIV_ROUND_UP(PAGE_ALIGN(eq->entries * eq->eqe_size), 5416 1 << page_shift); 5417 hns_roce_init_buf_region(®ion, hr_dev->caps.eqe_hop_num, 0, ba_num); 5418 5419 /* alloc a tmp list for storing eq buf address */ 5420 ret = hns_roce_alloc_buf_list(®ion, &buf_list, 1); 5421 if (ret) { 5422 dev_err(hr_dev->dev, "alloc eq buf_list error\n"); 5423 return ret; 5424 } 5425 5426 ba_num = hns_roce_get_kmem_bufs(hr_dev, buf_list, region.count, 5427 region.offset, &eq->buf); 5428 if (ba_num != region.count) { 5429 dev_err(hr_dev->dev, "get eqe buf err,expect %d,ret %d.\n", 5430 region.count, ba_num); 5431 ret = -ENOBUFS; 5432 goto done; 5433 } 5434 5435 hns_roce_mtr_init(&eq->mtr, PAGE_SHIFT + hr_dev->caps.eqe_ba_pg_sz, 5436 page_shift); 5437 ret = hns_roce_mtr_attach(hr_dev, &eq->mtr, &buf_list, ®ion, 1); 5438 if (ret) 5439 dev_err(hr_dev->dev, "mtr attach error for eqe\n"); 5440 5441 goto done; 5442 5443 hns_roce_mtr_cleanup(hr_dev, &eq->mtr); 5444 done: 5445 hns_roce_free_buf_list(&buf_list, 1); 5446 5447 return ret; 5448 } 5449 5450 static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) 5451 { 5452 struct hns_roce_buf *buf = &eq->buf; 5453 bool is_mhop = false; 5454 u32 page_shift; 5455 u32 mhop_num; 5456 u32 max_size; 5457 int ret; 5458 5459 page_shift = PAGE_SHIFT + hr_dev->caps.eqe_buf_pg_sz; 5460 mhop_num = hr_dev->caps.eqe_hop_num; 5461 if (!mhop_num) { 5462 max_size = 1 << page_shift; 5463 buf->size = max_size; 5464 } else if (mhop_num == HNS_ROCE_HOP_NUM_0) { 5465 max_size = eq->entries * eq->eqe_size; 5466 buf->size = max_size; 5467 } else { 5468 max_size = 1 << page_shift; 5469 buf->size = PAGE_ALIGN(eq->entries * eq->eqe_size); 5470 is_mhop = true; 5471 } 5472 5473 ret = hns_roce_buf_alloc(hr_dev, buf->size, max_size, buf, page_shift); 5474 if (ret) { 5475 dev_err(hr_dev->dev, "alloc eq buf error\n"); 5476 return ret; 5477 } 5478 5479 if (is_mhop) { 5480 ret = map_eq_buf(hr_dev, eq, page_shift); 5481 if (ret) { 5482 dev_err(hr_dev->dev, "map roce buf error\n"); 5483 goto err_alloc; 5484 } 5485 } 5486 5487 return 0; 5488 err_alloc: 5489 hns_roce_buf_free(hr_dev, buf->size, buf); 5490 return ret; 5491 } 5492 5493 static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev, 5494 struct hns_roce_eq *eq, 5495 unsigned int eq_cmd) 5496 { 5497 struct hns_roce_cmd_mailbox *mailbox; 5498 int ret; 5499 5500 /* Allocate mailbox memory */ 5501 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5502 if (IS_ERR(mailbox)) 5503 return PTR_ERR(mailbox); 5504 5505 ret = alloc_eq_buf(hr_dev, eq); 5506 if (ret) { 5507 ret = -ENOMEM; 5508 goto free_cmd_mbox; 5509 } 5510 hns_roce_config_eqc(hr_dev, eq, mailbox->buf); 5511 5512 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, eq->eqn, 0, 5513 eq_cmd, HNS_ROCE_CMD_TIMEOUT_MSECS); 5514 if (ret) { 5515 dev_err(hr_dev->dev, "[mailbox cmd] create eqc failed.\n"); 5516 goto err_cmd_mbox; 5517 } 5518 5519 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5520 5521 return 0; 5522 5523 err_cmd_mbox: 5524 free_eq_buf(hr_dev, eq); 5525 5526 free_cmd_mbox: 5527 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5528 5529 return ret; 5530 } 5531 5532 static int __hns_roce_request_irq(struct hns_roce_dev *hr_dev, int irq_num, 5533 int comp_num, int aeq_num, int other_num) 5534 { 5535 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; 5536 int i, j; 5537 int ret; 5538 5539 for (i = 0; i < irq_num; i++) { 5540 hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN, 5541 GFP_KERNEL); 5542 if (!hr_dev->irq_names[i]) { 5543 ret = -ENOMEM; 5544 goto err_kzalloc_failed; 5545 } 5546 } 5547 5548 /* irq contains: abnormal + AEQ + CEQ */ 5549 for (j = 0; j < other_num; j++) 5550 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN, 5551 "hns-abn-%d", j); 5552 5553 for (j = other_num; j < (other_num + aeq_num); j++) 5554 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN, 5555 "hns-aeq-%d", j - other_num); 5556 5557 for (j = (other_num + aeq_num); j < irq_num; j++) 5558 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN, 5559 "hns-ceq-%d", j - other_num - aeq_num); 5560 5561 for (j = 0; j < irq_num; j++) { 5562 if (j < other_num) 5563 ret = request_irq(hr_dev->irq[j], 5564 hns_roce_v2_msix_interrupt_abn, 5565 0, hr_dev->irq_names[j], hr_dev); 5566 5567 else if (j < (other_num + comp_num)) 5568 ret = request_irq(eq_table->eq[j - other_num].irq, 5569 hns_roce_v2_msix_interrupt_eq, 5570 0, hr_dev->irq_names[j + aeq_num], 5571 &eq_table->eq[j - other_num]); 5572 else 5573 ret = request_irq(eq_table->eq[j - other_num].irq, 5574 hns_roce_v2_msix_interrupt_eq, 5575 0, hr_dev->irq_names[j - comp_num], 5576 &eq_table->eq[j - other_num]); 5577 if (ret) { 5578 dev_err(hr_dev->dev, "Request irq error!\n"); 5579 goto err_request_failed; 5580 } 5581 } 5582 5583 return 0; 5584 5585 err_request_failed: 5586 for (j -= 1; j >= 0; j--) 5587 if (j < other_num) 5588 free_irq(hr_dev->irq[j], hr_dev); 5589 else 5590 free_irq(eq_table->eq[j - other_num].irq, 5591 &eq_table->eq[j - other_num]); 5592 5593 err_kzalloc_failed: 5594 for (i -= 1; i >= 0; i--) 5595 kfree(hr_dev->irq_names[i]); 5596 5597 return ret; 5598 } 5599 5600 static void __hns_roce_free_irq(struct hns_roce_dev *hr_dev) 5601 { 5602 int irq_num; 5603 int eq_num; 5604 int i; 5605 5606 eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; 5607 irq_num = eq_num + hr_dev->caps.num_other_vectors; 5608 5609 for (i = 0; i < hr_dev->caps.num_other_vectors; i++) 5610 free_irq(hr_dev->irq[i], hr_dev); 5611 5612 for (i = 0; i < eq_num; i++) 5613 free_irq(hr_dev->eq_table.eq[i].irq, &hr_dev->eq_table.eq[i]); 5614 5615 for (i = 0; i < irq_num; i++) 5616 kfree(hr_dev->irq_names[i]); 5617 } 5618 5619 static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev) 5620 { 5621 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; 5622 struct device *dev = hr_dev->dev; 5623 struct hns_roce_eq *eq; 5624 unsigned int eq_cmd; 5625 int irq_num; 5626 int eq_num; 5627 int other_num; 5628 int comp_num; 5629 int aeq_num; 5630 int i; 5631 int ret; 5632 5633 other_num = hr_dev->caps.num_other_vectors; 5634 comp_num = hr_dev->caps.num_comp_vectors; 5635 aeq_num = hr_dev->caps.num_aeq_vectors; 5636 5637 eq_num = comp_num + aeq_num; 5638 irq_num = eq_num + other_num; 5639 5640 eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL); 5641 if (!eq_table->eq) 5642 return -ENOMEM; 5643 5644 /* create eq */ 5645 for (i = 0; i < eq_num; i++) { 5646 eq = &eq_table->eq[i]; 5647 eq->hr_dev = hr_dev; 5648 eq->eqn = i; 5649 if (i < comp_num) { 5650 /* CEQ */ 5651 eq_cmd = HNS_ROCE_CMD_CREATE_CEQC; 5652 eq->type_flag = HNS_ROCE_CEQ; 5653 eq->entries = hr_dev->caps.ceqe_depth; 5654 eq->eqe_size = HNS_ROCE_CEQ_ENTRY_SIZE; 5655 eq->irq = hr_dev->irq[i + other_num + aeq_num]; 5656 eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM; 5657 eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL; 5658 } else { 5659 /* AEQ */ 5660 eq_cmd = HNS_ROCE_CMD_CREATE_AEQC; 5661 eq->type_flag = HNS_ROCE_AEQ; 5662 eq->entries = hr_dev->caps.aeqe_depth; 5663 eq->eqe_size = HNS_ROCE_AEQ_ENTRY_SIZE; 5664 eq->irq = hr_dev->irq[i - comp_num + other_num]; 5665 eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM; 5666 eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL; 5667 } 5668 5669 ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd); 5670 if (ret) { 5671 dev_err(dev, "eq create failed.\n"); 5672 goto err_create_eq_fail; 5673 } 5674 } 5675 5676 /* enable irq */ 5677 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE); 5678 5679 ret = __hns_roce_request_irq(hr_dev, irq_num, comp_num, 5680 aeq_num, other_num); 5681 if (ret) { 5682 dev_err(dev, "Request irq failed.\n"); 5683 goto err_request_irq_fail; 5684 } 5685 5686 hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", 0); 5687 if (!hr_dev->irq_workq) { 5688 dev_err(dev, "Create irq workqueue failed!\n"); 5689 ret = -ENOMEM; 5690 goto err_create_wq_fail; 5691 } 5692 5693 return 0; 5694 5695 err_create_wq_fail: 5696 __hns_roce_free_irq(hr_dev); 5697 5698 err_request_irq_fail: 5699 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE); 5700 5701 err_create_eq_fail: 5702 for (i -= 1; i >= 0; i--) 5703 free_eq_buf(hr_dev, &eq_table->eq[i]); 5704 kfree(eq_table->eq); 5705 5706 return ret; 5707 } 5708 5709 static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev) 5710 { 5711 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; 5712 int eq_num; 5713 int i; 5714 5715 eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; 5716 5717 /* Disable irq */ 5718 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE); 5719 5720 __hns_roce_free_irq(hr_dev); 5721 5722 for (i = 0; i < eq_num; i++) { 5723 hns_roce_v2_destroy_eqc(hr_dev, i); 5724 5725 free_eq_buf(hr_dev, &eq_table->eq[i]); 5726 } 5727 5728 kfree(eq_table->eq); 5729 5730 flush_workqueue(hr_dev->irq_workq); 5731 destroy_workqueue(hr_dev->irq_workq); 5732 } 5733 5734 static void hns_roce_v2_write_srqc(struct hns_roce_dev *hr_dev, 5735 struct hns_roce_srq *srq, u32 pdn, u16 xrcd, 5736 u32 cqn, void *mb_buf, u64 *mtts_wqe, 5737 u64 *mtts_idx, dma_addr_t dma_handle_wqe, 5738 dma_addr_t dma_handle_idx) 5739 { 5740 struct hns_roce_srq_context *srq_context; 5741 5742 srq_context = mb_buf; 5743 memset(srq_context, 0, sizeof(*srq_context)); 5744 5745 roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQ_ST_M, 5746 SRQC_BYTE_4_SRQ_ST_S, 1); 5747 5748 roce_set_field(srq_context->byte_4_srqn_srqst, 5749 SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M, 5750 SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S, 5751 (hr_dev->caps.srqwqe_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : 5752 hr_dev->caps.srqwqe_hop_num)); 5753 roce_set_field(srq_context->byte_4_srqn_srqst, 5754 SRQC_BYTE_4_SRQ_SHIFT_M, SRQC_BYTE_4_SRQ_SHIFT_S, 5755 ilog2(srq->wqe_cnt)); 5756 5757 roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQN_M, 5758 SRQC_BYTE_4_SRQN_S, srq->srqn); 5759 5760 roce_set_field(srq_context->byte_8_limit_wl, SRQC_BYTE_8_SRQ_LIMIT_WL_M, 5761 SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0); 5762 5763 roce_set_field(srq_context->byte_12_xrcd, SRQC_BYTE_12_SRQ_XRCD_M, 5764 SRQC_BYTE_12_SRQ_XRCD_S, xrcd); 5765 5766 srq_context->wqe_bt_ba = cpu_to_le32((u32)(dma_handle_wqe >> 3)); 5767 5768 roce_set_field(srq_context->byte_24_wqe_bt_ba, 5769 SRQC_BYTE_24_SRQ_WQE_BT_BA_M, 5770 SRQC_BYTE_24_SRQ_WQE_BT_BA_S, 5771 dma_handle_wqe >> 35); 5772 5773 roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_PD_M, 5774 SRQC_BYTE_28_PD_S, pdn); 5775 roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_RQWS_M, 5776 SRQC_BYTE_28_RQWS_S, srq->max_gs <= 0 ? 0 : 5777 fls(srq->max_gs - 1)); 5778 5779 srq_context->idx_bt_ba = cpu_to_le32(dma_handle_idx >> 3); 5780 roce_set_field(srq_context->rsv_idx_bt_ba, 5781 SRQC_BYTE_36_SRQ_IDX_BT_BA_M, 5782 SRQC_BYTE_36_SRQ_IDX_BT_BA_S, 5783 dma_handle_idx >> 35); 5784 5785 srq_context->idx_cur_blk_addr = 5786 cpu_to_le32(mtts_idx[0] >> PAGE_ADDR_SHIFT); 5787 roce_set_field(srq_context->byte_44_idxbufpgsz_addr, 5788 SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M, 5789 SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S, 5790 mtts_idx[0] >> (32 + PAGE_ADDR_SHIFT)); 5791 roce_set_field(srq_context->byte_44_idxbufpgsz_addr, 5792 SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M, 5793 SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S, 5794 hr_dev->caps.idx_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : 5795 hr_dev->caps.idx_hop_num); 5796 5797 roce_set_field(srq_context->byte_44_idxbufpgsz_addr, 5798 SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M, 5799 SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S, 5800 hr_dev->caps.idx_ba_pg_sz + PG_SHIFT_OFFSET); 5801 roce_set_field(srq_context->byte_44_idxbufpgsz_addr, 5802 SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M, 5803 SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S, 5804 hr_dev->caps.idx_buf_pg_sz + PG_SHIFT_OFFSET); 5805 5806 srq_context->idx_nxt_blk_addr = 5807 cpu_to_le32(mtts_idx[1] >> PAGE_ADDR_SHIFT); 5808 roce_set_field(srq_context->rsv_idxnxtblkaddr, 5809 SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M, 5810 SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S, 5811 mtts_idx[1] >> (32 + PAGE_ADDR_SHIFT)); 5812 roce_set_field(srq_context->byte_56_xrc_cqn, 5813 SRQC_BYTE_56_SRQ_XRC_CQN_M, SRQC_BYTE_56_SRQ_XRC_CQN_S, 5814 cqn); 5815 roce_set_field(srq_context->byte_56_xrc_cqn, 5816 SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M, 5817 SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S, 5818 hr_dev->caps.srqwqe_ba_pg_sz + PG_SHIFT_OFFSET); 5819 roce_set_field(srq_context->byte_56_xrc_cqn, 5820 SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M, 5821 SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S, 5822 hr_dev->caps.srqwqe_buf_pg_sz + PG_SHIFT_OFFSET); 5823 5824 roce_set_bit(srq_context->db_record_addr_record_en, 5825 SRQC_BYTE_60_SRQ_RECORD_EN_S, 0); 5826 } 5827 5828 static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq, 5829 struct ib_srq_attr *srq_attr, 5830 enum ib_srq_attr_mask srq_attr_mask, 5831 struct ib_udata *udata) 5832 { 5833 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); 5834 struct hns_roce_srq *srq = to_hr_srq(ibsrq); 5835 struct hns_roce_srq_context *srq_context; 5836 struct hns_roce_srq_context *srqc_mask; 5837 struct hns_roce_cmd_mailbox *mailbox; 5838 int ret; 5839 5840 if (srq_attr_mask & IB_SRQ_LIMIT) { 5841 if (srq_attr->srq_limit >= srq->wqe_cnt) 5842 return -EINVAL; 5843 5844 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5845 if (IS_ERR(mailbox)) 5846 return PTR_ERR(mailbox); 5847 5848 srq_context = mailbox->buf; 5849 srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1; 5850 5851 memset(srqc_mask, 0xff, sizeof(*srqc_mask)); 5852 5853 roce_set_field(srq_context->byte_8_limit_wl, 5854 SRQC_BYTE_8_SRQ_LIMIT_WL_M, 5855 SRQC_BYTE_8_SRQ_LIMIT_WL_S, srq_attr->srq_limit); 5856 roce_set_field(srqc_mask->byte_8_limit_wl, 5857 SRQC_BYTE_8_SRQ_LIMIT_WL_M, 5858 SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0); 5859 5860 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, srq->srqn, 0, 5861 HNS_ROCE_CMD_MODIFY_SRQC, 5862 HNS_ROCE_CMD_TIMEOUT_MSECS); 5863 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5864 if (ret) { 5865 ibdev_err(&hr_dev->ib_dev, 5866 "failed to process cmd when modifying SRQ, ret = %d\n", 5867 ret); 5868 return ret; 5869 } 5870 } 5871 5872 return 0; 5873 } 5874 5875 static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) 5876 { 5877 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); 5878 struct hns_roce_srq *srq = to_hr_srq(ibsrq); 5879 struct hns_roce_srq_context *srq_context; 5880 struct hns_roce_cmd_mailbox *mailbox; 5881 int limit_wl; 5882 int ret; 5883 5884 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5885 if (IS_ERR(mailbox)) 5886 return PTR_ERR(mailbox); 5887 5888 srq_context = mailbox->buf; 5889 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, srq->srqn, 0, 5890 HNS_ROCE_CMD_QUERY_SRQC, 5891 HNS_ROCE_CMD_TIMEOUT_MSECS); 5892 if (ret) { 5893 ibdev_err(&hr_dev->ib_dev, 5894 "failed to process cmd when querying SRQ, ret = %d\n", 5895 ret); 5896 goto out; 5897 } 5898 5899 limit_wl = roce_get_field(srq_context->byte_8_limit_wl, 5900 SRQC_BYTE_8_SRQ_LIMIT_WL_M, 5901 SRQC_BYTE_8_SRQ_LIMIT_WL_S); 5902 5903 attr->srq_limit = limit_wl; 5904 attr->max_wr = srq->wqe_cnt - 1; 5905 attr->max_sge = srq->max_gs; 5906 5907 memcpy(srq_context, mailbox->buf, sizeof(*srq_context)); 5908 5909 out: 5910 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5911 return ret; 5912 } 5913 5914 static int find_empty_entry(struct hns_roce_idx_que *idx_que, 5915 unsigned long size) 5916 { 5917 int wqe_idx; 5918 5919 if (unlikely(bitmap_full(idx_que->bitmap, size))) 5920 return -ENOSPC; 5921 5922 wqe_idx = find_first_zero_bit(idx_que->bitmap, size); 5923 5924 bitmap_set(idx_que->bitmap, wqe_idx, 1); 5925 5926 return wqe_idx; 5927 } 5928 5929 static void fill_idx_queue(struct hns_roce_idx_que *idx_que, 5930 int cur_idx, int wqe_idx) 5931 { 5932 unsigned int *addr; 5933 5934 addr = (unsigned int *)hns_roce_buf_offset(&idx_que->idx_buf, 5935 cur_idx * idx_que->entry_sz); 5936 *addr = wqe_idx; 5937 } 5938 5939 static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq, 5940 const struct ib_recv_wr *wr, 5941 const struct ib_recv_wr **bad_wr) 5942 { 5943 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); 5944 struct hns_roce_srq *srq = to_hr_srq(ibsrq); 5945 struct hns_roce_v2_wqe_data_seg *dseg; 5946 struct hns_roce_v2_db srq_db; 5947 unsigned long flags; 5948 int ret = 0; 5949 int wqe_idx; 5950 void *wqe; 5951 int nreq; 5952 int ind; 5953 int i; 5954 5955 spin_lock_irqsave(&srq->lock, flags); 5956 5957 ind = srq->head & (srq->wqe_cnt - 1); 5958 5959 for (nreq = 0; wr; ++nreq, wr = wr->next) { 5960 if (unlikely(wr->num_sge > srq->max_gs)) { 5961 ret = -EINVAL; 5962 *bad_wr = wr; 5963 break; 5964 } 5965 5966 if (unlikely(srq->head == srq->tail)) { 5967 ret = -ENOMEM; 5968 *bad_wr = wr; 5969 break; 5970 } 5971 5972 wqe_idx = find_empty_entry(&srq->idx_que, srq->wqe_cnt); 5973 if (wqe_idx < 0) { 5974 ret = -ENOMEM; 5975 *bad_wr = wr; 5976 break; 5977 } 5978 5979 fill_idx_queue(&srq->idx_que, ind, wqe_idx); 5980 wqe = get_srq_wqe(srq, wqe_idx); 5981 dseg = (struct hns_roce_v2_wqe_data_seg *)wqe; 5982 5983 for (i = 0; i < wr->num_sge; ++i) { 5984 dseg[i].len = cpu_to_le32(wr->sg_list[i].length); 5985 dseg[i].lkey = cpu_to_le32(wr->sg_list[i].lkey); 5986 dseg[i].addr = cpu_to_le64(wr->sg_list[i].addr); 5987 } 5988 5989 if (i < srq->max_gs) { 5990 dseg[i].len = 0; 5991 dseg[i].lkey = cpu_to_le32(0x100); 5992 dseg[i].addr = 0; 5993 } 5994 5995 srq->wrid[wqe_idx] = wr->wr_id; 5996 ind = (ind + 1) & (srq->wqe_cnt - 1); 5997 } 5998 5999 if (likely(nreq)) { 6000 srq->head += nreq; 6001 6002 /* 6003 * Make sure that descriptors are written before 6004 * doorbell record. 6005 */ 6006 wmb(); 6007 6008 srq_db.byte_4 = 6009 cpu_to_le32(HNS_ROCE_V2_SRQ_DB << V2_DB_BYTE_4_CMD_S | 6010 (srq->srqn & V2_DB_BYTE_4_TAG_M)); 6011 srq_db.parameter = cpu_to_le32(srq->head); 6012 6013 hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l); 6014 6015 } 6016 6017 spin_unlock_irqrestore(&srq->lock, flags); 6018 6019 return ret; 6020 } 6021 6022 static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = { 6023 .query_cqc_info = hns_roce_v2_query_cqc_info, 6024 }; 6025 6026 static const struct ib_device_ops hns_roce_v2_dev_ops = { 6027 .destroy_qp = hns_roce_v2_destroy_qp, 6028 .modify_cq = hns_roce_v2_modify_cq, 6029 .poll_cq = hns_roce_v2_poll_cq, 6030 .post_recv = hns_roce_v2_post_recv, 6031 .post_send = hns_roce_v2_post_send, 6032 .query_qp = hns_roce_v2_query_qp, 6033 .req_notify_cq = hns_roce_v2_req_notify_cq, 6034 }; 6035 6036 static const struct ib_device_ops hns_roce_v2_dev_srq_ops = { 6037 .modify_srq = hns_roce_v2_modify_srq, 6038 .post_srq_recv = hns_roce_v2_post_srq_recv, 6039 .query_srq = hns_roce_v2_query_srq, 6040 }; 6041 6042 static const struct hns_roce_hw hns_roce_hw_v2 = { 6043 .cmq_init = hns_roce_v2_cmq_init, 6044 .cmq_exit = hns_roce_v2_cmq_exit, 6045 .hw_profile = hns_roce_v2_profile, 6046 .hw_init = hns_roce_v2_init, 6047 .hw_exit = hns_roce_v2_exit, 6048 .post_mbox = hns_roce_v2_post_mbox, 6049 .chk_mbox = hns_roce_v2_chk_mbox, 6050 .rst_prc_mbox = hns_roce_v2_rst_process_cmd, 6051 .set_gid = hns_roce_v2_set_gid, 6052 .set_mac = hns_roce_v2_set_mac, 6053 .write_mtpt = hns_roce_v2_write_mtpt, 6054 .rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt, 6055 .frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt, 6056 .mw_write_mtpt = hns_roce_v2_mw_write_mtpt, 6057 .write_cqc = hns_roce_v2_write_cqc, 6058 .set_hem = hns_roce_v2_set_hem, 6059 .clear_hem = hns_roce_v2_clear_hem, 6060 .modify_qp = hns_roce_v2_modify_qp, 6061 .query_qp = hns_roce_v2_query_qp, 6062 .destroy_qp = hns_roce_v2_destroy_qp, 6063 .qp_flow_control_init = hns_roce_v2_qp_flow_control_init, 6064 .modify_cq = hns_roce_v2_modify_cq, 6065 .post_send = hns_roce_v2_post_send, 6066 .post_recv = hns_roce_v2_post_recv, 6067 .req_notify_cq = hns_roce_v2_req_notify_cq, 6068 .poll_cq = hns_roce_v2_poll_cq, 6069 .init_eq = hns_roce_v2_init_eq_table, 6070 .cleanup_eq = hns_roce_v2_cleanup_eq_table, 6071 .write_srqc = hns_roce_v2_write_srqc, 6072 .modify_srq = hns_roce_v2_modify_srq, 6073 .query_srq = hns_roce_v2_query_srq, 6074 .post_srq_recv = hns_roce_v2_post_srq_recv, 6075 .hns_roce_dev_ops = &hns_roce_v2_dev_ops, 6076 .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops, 6077 }; 6078 6079 static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = { 6080 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0}, 6081 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0}, 6082 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0}, 6083 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0}, 6084 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0}, 6085 /* required last entry */ 6086 {0, } 6087 }; 6088 6089 MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl); 6090 6091 static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev, 6092 struct hnae3_handle *handle) 6093 { 6094 struct hns_roce_v2_priv *priv = hr_dev->priv; 6095 int i; 6096 6097 hr_dev->pci_dev = handle->pdev; 6098 hr_dev->dev = &handle->pdev->dev; 6099 hr_dev->hw = &hns_roce_hw_v2; 6100 hr_dev->dfx = &hns_roce_dfx_hw_v2; 6101 hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG; 6102 hr_dev->odb_offset = hr_dev->sdb_offset; 6103 6104 /* Get info from NIC driver. */ 6105 hr_dev->reg_base = handle->rinfo.roce_io_base; 6106 hr_dev->caps.num_ports = 1; 6107 hr_dev->iboe.netdevs[0] = handle->rinfo.netdev; 6108 hr_dev->iboe.phy_port[0] = 0; 6109 6110 addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid, 6111 hr_dev->iboe.netdevs[0]->dev_addr); 6112 6113 for (i = 0; i < HNS_ROCE_V2_MAX_IRQ_NUM; i++) 6114 hr_dev->irq[i] = pci_irq_vector(handle->pdev, 6115 i + handle->rinfo.base_vector); 6116 6117 /* cmd issue mode: 0 is poll, 1 is event */ 6118 hr_dev->cmd_mod = 1; 6119 hr_dev->loop_idc = 0; 6120 6121 hr_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle); 6122 priv->handle = handle; 6123 } 6124 6125 static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) 6126 { 6127 struct hns_roce_dev *hr_dev; 6128 int ret; 6129 6130 hr_dev = ib_alloc_device(hns_roce_dev, ib_dev); 6131 if (!hr_dev) 6132 return -ENOMEM; 6133 6134 hr_dev->priv = kzalloc(sizeof(struct hns_roce_v2_priv), GFP_KERNEL); 6135 if (!hr_dev->priv) { 6136 ret = -ENOMEM; 6137 goto error_failed_kzalloc; 6138 } 6139 6140 hns_roce_hw_v2_get_cfg(hr_dev, handle); 6141 6142 ret = hns_roce_init(hr_dev); 6143 if (ret) { 6144 dev_err(hr_dev->dev, "RoCE Engine init failed!\n"); 6145 goto error_failed_get_cfg; 6146 } 6147 6148 handle->priv = hr_dev; 6149 6150 return 0; 6151 6152 error_failed_get_cfg: 6153 kfree(hr_dev->priv); 6154 6155 error_failed_kzalloc: 6156 ib_dealloc_device(&hr_dev->ib_dev); 6157 6158 return ret; 6159 } 6160 6161 static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, 6162 bool reset) 6163 { 6164 struct hns_roce_dev *hr_dev = (struct hns_roce_dev *)handle->priv; 6165 6166 if (!hr_dev) 6167 return; 6168 6169 handle->priv = NULL; 6170 6171 hr_dev->state = HNS_ROCE_DEVICE_STATE_UNINIT; 6172 hns_roce_handle_device_err(hr_dev); 6173 6174 hns_roce_exit(hr_dev); 6175 kfree(hr_dev->priv); 6176 ib_dealloc_device(&hr_dev->ib_dev); 6177 } 6178 6179 static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) 6180 { 6181 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 6182 const struct pci_device_id *id; 6183 struct device *dev = &handle->pdev->dev; 6184 int ret; 6185 6186 handle->rinfo.instance_state = HNS_ROCE_STATE_INIT; 6187 6188 if (ops->ae_dev_resetting(handle) || ops->get_hw_reset_stat(handle)) { 6189 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; 6190 goto reset_chk_err; 6191 } 6192 6193 id = pci_match_id(hns_roce_hw_v2_pci_tbl, handle->pdev); 6194 if (!id) 6195 return 0; 6196 6197 ret = __hns_roce_hw_v2_init_instance(handle); 6198 if (ret) { 6199 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; 6200 dev_err(dev, "RoCE instance init failed! ret = %d\n", ret); 6201 if (ops->ae_dev_resetting(handle) || 6202 ops->get_hw_reset_stat(handle)) 6203 goto reset_chk_err; 6204 else 6205 return ret; 6206 } 6207 6208 handle->rinfo.instance_state = HNS_ROCE_STATE_INITED; 6209 6210 6211 return 0; 6212 6213 reset_chk_err: 6214 dev_err(dev, "Device is busy in resetting state.\n" 6215 "please retry later.\n"); 6216 6217 return -EBUSY; 6218 } 6219 6220 static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, 6221 bool reset) 6222 { 6223 if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) 6224 return; 6225 6226 handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT; 6227 6228 __hns_roce_hw_v2_uninit_instance(handle, reset); 6229 6230 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; 6231 } 6232 static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle) 6233 { 6234 struct hns_roce_dev *hr_dev; 6235 6236 if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) { 6237 set_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state); 6238 return 0; 6239 } 6240 6241 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN; 6242 clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state); 6243 6244 hr_dev = (struct hns_roce_dev *)handle->priv; 6245 if (!hr_dev) 6246 return 0; 6247 6248 hr_dev->is_reset = true; 6249 hr_dev->active = false; 6250 hr_dev->dis_db = true; 6251 6252 hr_dev->state = HNS_ROCE_DEVICE_STATE_RST_DOWN; 6253 6254 return 0; 6255 } 6256 6257 static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle) 6258 { 6259 struct device *dev = &handle->pdev->dev; 6260 int ret; 6261 6262 if (test_and_clear_bit(HNS_ROCE_RST_DIRECT_RETURN, 6263 &handle->rinfo.state)) { 6264 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED; 6265 return 0; 6266 } 6267 6268 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INIT; 6269 6270 dev_info(&handle->pdev->dev, "In reset process RoCE client reinit.\n"); 6271 ret = __hns_roce_hw_v2_init_instance(handle); 6272 if (ret) { 6273 /* when reset notify type is HNAE3_INIT_CLIENT In reset notify 6274 * callback function, RoCE Engine reinitialize. If RoCE reinit 6275 * failed, we should inform NIC driver. 6276 */ 6277 handle->priv = NULL; 6278 dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret); 6279 } else { 6280 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED; 6281 dev_info(dev, "Reset done, RoCE client reinit finished.\n"); 6282 } 6283 6284 return ret; 6285 } 6286 6287 static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle) 6288 { 6289 if (test_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state)) 6290 return 0; 6291 6292 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT; 6293 dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n"); 6294 msleep(HNS_ROCE_V2_HW_RST_UNINT_DELAY); 6295 __hns_roce_hw_v2_uninit_instance(handle, false); 6296 6297 return 0; 6298 } 6299 6300 static int hns_roce_hw_v2_reset_notify(struct hnae3_handle *handle, 6301 enum hnae3_reset_notify_type type) 6302 { 6303 int ret = 0; 6304 6305 switch (type) { 6306 case HNAE3_DOWN_CLIENT: 6307 ret = hns_roce_hw_v2_reset_notify_down(handle); 6308 break; 6309 case HNAE3_INIT_CLIENT: 6310 ret = hns_roce_hw_v2_reset_notify_init(handle); 6311 break; 6312 case HNAE3_UNINIT_CLIENT: 6313 ret = hns_roce_hw_v2_reset_notify_uninit(handle); 6314 break; 6315 default: 6316 break; 6317 } 6318 6319 return ret; 6320 } 6321 6322 static const struct hnae3_client_ops hns_roce_hw_v2_ops = { 6323 .init_instance = hns_roce_hw_v2_init_instance, 6324 .uninit_instance = hns_roce_hw_v2_uninit_instance, 6325 .reset_notify = hns_roce_hw_v2_reset_notify, 6326 }; 6327 6328 static struct hnae3_client hns_roce_hw_v2_client = { 6329 .name = "hns_roce_hw_v2", 6330 .type = HNAE3_CLIENT_ROCE, 6331 .ops = &hns_roce_hw_v2_ops, 6332 }; 6333 6334 static int __init hns_roce_hw_v2_init(void) 6335 { 6336 return hnae3_register_client(&hns_roce_hw_v2_client); 6337 } 6338 6339 static void __exit hns_roce_hw_v2_exit(void) 6340 { 6341 hnae3_unregister_client(&hns_roce_hw_v2_client); 6342 } 6343 6344 module_init(hns_roce_hw_v2_init); 6345 module_exit(hns_roce_hw_v2_exit); 6346 6347 MODULE_LICENSE("Dual BSD/GPL"); 6348 MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>"); 6349 MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>"); 6350 MODULE_AUTHOR("Shaobo Xu <xushaobo2@huawei.com>"); 6351 MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE Driver"); 6352