1 /* 2 * Broadcom NetXtreme-E RoCE driver. 3 * 4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term 5 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the 22 * distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * Description: Fast Path Operators 37 */ 38 39 #include <linux/interrupt.h> 40 #include <linux/spinlock.h> 41 #include <linux/sched.h> 42 #include <linux/slab.h> 43 #include <linux/pci.h> 44 #include <linux/prefetch.h> 45 46 #include "roce_hsi.h" 47 48 #include "qplib_res.h" 49 #include "qplib_rcfw.h" 50 #include "qplib_sp.h" 51 #include "qplib_fp.h" 52 53 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq); 54 55 static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res, 56 struct bnxt_qplib_qp *qp) 57 { 58 struct bnxt_qplib_q *rq = &qp->rq; 59 struct bnxt_qplib_q *sq = &qp->sq; 60 61 if (qp->rq_hdr_buf) 62 dma_free_coherent(&res->pdev->dev, 63 rq->hwq.max_elements * qp->rq_hdr_buf_size, 64 qp->rq_hdr_buf, qp->rq_hdr_buf_map); 65 if (qp->sq_hdr_buf) 66 dma_free_coherent(&res->pdev->dev, 67 sq->hwq.max_elements * qp->sq_hdr_buf_size, 68 qp->sq_hdr_buf, qp->sq_hdr_buf_map); 69 qp->rq_hdr_buf = NULL; 70 qp->sq_hdr_buf = NULL; 71 qp->rq_hdr_buf_map = 0; 72 qp->sq_hdr_buf_map = 0; 73 qp->sq_hdr_buf_size = 0; 74 qp->rq_hdr_buf_size = 0; 75 } 76 77 static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res, 78 struct bnxt_qplib_qp *qp) 79 { 80 struct bnxt_qplib_q *rq = &qp->rq; 81 struct bnxt_qplib_q *sq = &qp->rq; 82 int rc = 0; 83 84 if (qp->sq_hdr_buf_size && sq->hwq.max_elements) { 85 qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, 86 sq->hwq.max_elements * 87 qp->sq_hdr_buf_size, 88 &qp->sq_hdr_buf_map, GFP_KERNEL); 89 if (!qp->sq_hdr_buf) { 90 rc = -ENOMEM; 91 dev_err(&res->pdev->dev, 92 "QPLIB: Failed to create sq_hdr_buf"); 93 goto fail; 94 } 95 } 96 97 if (qp->rq_hdr_buf_size && rq->hwq.max_elements) { 98 qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, 99 rq->hwq.max_elements * 100 qp->rq_hdr_buf_size, 101 &qp->rq_hdr_buf_map, 102 GFP_KERNEL); 103 if (!qp->rq_hdr_buf) { 104 rc = -ENOMEM; 105 dev_err(&res->pdev->dev, 106 "QPLIB: Failed to create rq_hdr_buf"); 107 goto fail; 108 } 109 } 110 return 0; 111 112 fail: 113 bnxt_qplib_free_qp_hdr_buf(res, qp); 114 return rc; 115 } 116 117 static void bnxt_qplib_service_nq(unsigned long data) 118 { 119 struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data; 120 struct bnxt_qplib_hwq *hwq = &nq->hwq; 121 struct nq_base *nqe, **nq_ptr; 122 int num_cqne_processed = 0; 123 u32 sw_cons, raw_cons; 124 u16 type; 125 int budget = nq->budget; 126 u64 q_handle; 127 128 /* Service the NQ until empty */ 129 raw_cons = hwq->cons; 130 while (budget--) { 131 sw_cons = HWQ_CMP(raw_cons, hwq); 132 nq_ptr = (struct nq_base **)hwq->pbl_ptr; 133 nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]; 134 if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements)) 135 break; 136 137 type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK; 138 switch (type) { 139 case NQ_BASE_TYPE_CQ_NOTIFICATION: 140 { 141 struct nq_cn *nqcne = (struct nq_cn *)nqe; 142 143 q_handle = le32_to_cpu(nqcne->cq_handle_low); 144 q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) 145 << 32; 146 bnxt_qplib_arm_cq_enable((struct bnxt_qplib_cq *) 147 ((unsigned long)q_handle)); 148 if (!nq->cqn_handler(nq, (struct bnxt_qplib_cq *) 149 ((unsigned long)q_handle))) 150 num_cqne_processed++; 151 else 152 dev_warn(&nq->pdev->dev, 153 "QPLIB: cqn - type 0x%x not handled", 154 type); 155 break; 156 } 157 case NQ_BASE_TYPE_DBQ_EVENT: 158 break; 159 default: 160 dev_warn(&nq->pdev->dev, 161 "QPLIB: nqe with type = 0x%x not handled", 162 type); 163 break; 164 } 165 raw_cons++; 166 } 167 if (hwq->cons != raw_cons) { 168 hwq->cons = raw_cons; 169 NQ_DB_REARM(nq->bar_reg_iomem, hwq->cons, hwq->max_elements); 170 } 171 } 172 173 static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance) 174 { 175 struct bnxt_qplib_nq *nq = dev_instance; 176 struct bnxt_qplib_hwq *hwq = &nq->hwq; 177 struct nq_base **nq_ptr; 178 u32 sw_cons; 179 180 /* Prefetch the NQ element */ 181 sw_cons = HWQ_CMP(hwq->cons, hwq); 182 nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr; 183 prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]); 184 185 /* Fan out to CPU affinitized kthreads? */ 186 tasklet_schedule(&nq->worker); 187 188 return IRQ_HANDLED; 189 } 190 191 void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq) 192 { 193 /* Make sure the HW is stopped! */ 194 synchronize_irq(nq->vector); 195 tasklet_disable(&nq->worker); 196 tasklet_kill(&nq->worker); 197 198 if (nq->requested) { 199 free_irq(nq->vector, nq); 200 nq->requested = false; 201 } 202 if (nq->bar_reg_iomem) 203 iounmap(nq->bar_reg_iomem); 204 nq->bar_reg_iomem = NULL; 205 206 nq->cqn_handler = NULL; 207 nq->srqn_handler = NULL; 208 nq->vector = 0; 209 } 210 211 int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq, 212 int msix_vector, int bar_reg_offset, 213 int (*cqn_handler)(struct bnxt_qplib_nq *nq, 214 struct bnxt_qplib_cq *), 215 int (*srqn_handler)(struct bnxt_qplib_nq *nq, 216 void *, u8 event)) 217 { 218 resource_size_t nq_base; 219 int rc; 220 221 nq->pdev = pdev; 222 nq->vector = msix_vector; 223 224 nq->cqn_handler = cqn_handler; 225 226 nq->srqn_handler = srqn_handler; 227 228 tasklet_init(&nq->worker, bnxt_qplib_service_nq, (unsigned long)nq); 229 230 nq->requested = false; 231 rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, "bnxt_qplib_nq", nq); 232 if (rc) { 233 dev_err(&nq->pdev->dev, 234 "Failed to request IRQ for NQ: %#x", rc); 235 bnxt_qplib_disable_nq(nq); 236 goto fail; 237 } 238 nq->requested = true; 239 nq->bar_reg = NQ_CONS_PCI_BAR_REGION; 240 nq->bar_reg_off = bar_reg_offset; 241 nq_base = pci_resource_start(pdev, nq->bar_reg); 242 if (!nq_base) { 243 rc = -ENOMEM; 244 goto fail; 245 } 246 nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 4); 247 if (!nq->bar_reg_iomem) { 248 rc = -ENOMEM; 249 goto fail; 250 } 251 NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements); 252 253 return 0; 254 fail: 255 bnxt_qplib_disable_nq(nq); 256 return rc; 257 } 258 259 void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq) 260 { 261 if (nq->hwq.max_elements) 262 bnxt_qplib_free_hwq(nq->pdev, &nq->hwq); 263 } 264 265 int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq) 266 { 267 nq->pdev = pdev; 268 if (!nq->hwq.max_elements || 269 nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT) 270 nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT; 271 272 if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0, 273 &nq->hwq.max_elements, 274 BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0, 275 PAGE_SIZE, HWQ_TYPE_L2_CMPL)) 276 return -ENOMEM; 277 278 nq->budget = 8; 279 return 0; 280 } 281 282 /* QP */ 283 int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) 284 { 285 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 286 struct cmdq_create_qp1 req; 287 struct creq_create_qp1_resp resp; 288 struct bnxt_qplib_pbl *pbl; 289 struct bnxt_qplib_q *sq = &qp->sq; 290 struct bnxt_qplib_q *rq = &qp->rq; 291 int rc; 292 u16 cmd_flags = 0; 293 u32 qp_flags = 0; 294 295 RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags); 296 297 /* General */ 298 req.type = qp->type; 299 req.dpi = cpu_to_le32(qp->dpi->dpi); 300 req.qp_handle = cpu_to_le64(qp->qp_handle); 301 302 /* SQ */ 303 sq->hwq.max_elements = sq->max_wqe; 304 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL, 0, 305 &sq->hwq.max_elements, 306 BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0, 307 PAGE_SIZE, HWQ_TYPE_QUEUE); 308 if (rc) 309 goto exit; 310 311 sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL); 312 if (!sq->swq) { 313 rc = -ENOMEM; 314 goto fail_sq; 315 } 316 pbl = &sq->hwq.pbl[PBL_LVL_0]; 317 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); 318 req.sq_pg_size_sq_lvl = 319 ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK) 320 << CMDQ_CREATE_QP1_SQ_LVL_SFT) | 321 (pbl->pg_size == ROCE_PG_SIZE_4K ? 322 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K : 323 pbl->pg_size == ROCE_PG_SIZE_8K ? 324 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K : 325 pbl->pg_size == ROCE_PG_SIZE_64K ? 326 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K : 327 pbl->pg_size == ROCE_PG_SIZE_2M ? 328 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M : 329 pbl->pg_size == ROCE_PG_SIZE_8M ? 330 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M : 331 pbl->pg_size == ROCE_PG_SIZE_1G ? 332 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G : 333 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K); 334 335 if (qp->scq) 336 req.scq_cid = cpu_to_le32(qp->scq->id); 337 338 qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE; 339 340 /* RQ */ 341 if (rq->max_wqe) { 342 rq->hwq.max_elements = qp->rq.max_wqe; 343 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL, 0, 344 &rq->hwq.max_elements, 345 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0, 346 PAGE_SIZE, HWQ_TYPE_QUEUE); 347 if (rc) 348 goto fail_sq; 349 350 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq), 351 GFP_KERNEL); 352 if (!rq->swq) { 353 rc = -ENOMEM; 354 goto fail_rq; 355 } 356 pbl = &rq->hwq.pbl[PBL_LVL_0]; 357 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); 358 req.rq_pg_size_rq_lvl = 359 ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) << 360 CMDQ_CREATE_QP1_RQ_LVL_SFT) | 361 (pbl->pg_size == ROCE_PG_SIZE_4K ? 362 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K : 363 pbl->pg_size == ROCE_PG_SIZE_8K ? 364 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K : 365 pbl->pg_size == ROCE_PG_SIZE_64K ? 366 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K : 367 pbl->pg_size == ROCE_PG_SIZE_2M ? 368 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M : 369 pbl->pg_size == ROCE_PG_SIZE_8M ? 370 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M : 371 pbl->pg_size == ROCE_PG_SIZE_1G ? 372 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G : 373 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K); 374 if (qp->rcq) 375 req.rcq_cid = cpu_to_le32(qp->rcq->id); 376 } 377 378 /* Header buffer - allow hdr_buf pass in */ 379 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); 380 if (rc) { 381 rc = -ENOMEM; 382 goto fail; 383 } 384 req.qp_flags = cpu_to_le32(qp_flags); 385 req.sq_size = cpu_to_le32(sq->hwq.max_elements); 386 req.rq_size = cpu_to_le32(rq->hwq.max_elements); 387 388 req.sq_fwo_sq_sge = 389 cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) << 390 CMDQ_CREATE_QP1_SQ_SGE_SFT); 391 req.rq_fwo_rq_sge = 392 cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) << 393 CMDQ_CREATE_QP1_RQ_SGE_SFT); 394 395 req.pd_id = cpu_to_le32(qp->pd->id); 396 397 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 398 (void *)&resp, NULL, 0); 399 if (rc) 400 goto fail; 401 402 qp->id = le32_to_cpu(resp.xid); 403 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; 404 sq->flush_in_progress = false; 405 rq->flush_in_progress = false; 406 407 return 0; 408 409 fail: 410 bnxt_qplib_free_qp_hdr_buf(res, qp); 411 fail_rq: 412 bnxt_qplib_free_hwq(res->pdev, &rq->hwq); 413 kfree(rq->swq); 414 fail_sq: 415 bnxt_qplib_free_hwq(res->pdev, &sq->hwq); 416 kfree(sq->swq); 417 exit: 418 return rc; 419 } 420 421 int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) 422 { 423 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 424 struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr; 425 struct cmdq_create_qp req; 426 struct creq_create_qp_resp resp; 427 struct bnxt_qplib_pbl *pbl; 428 struct sq_psn_search **psn_search_ptr; 429 unsigned long int psn_search, poff = 0; 430 struct bnxt_qplib_q *sq = &qp->sq; 431 struct bnxt_qplib_q *rq = &qp->rq; 432 struct bnxt_qplib_hwq *xrrq; 433 int i, rc, req_size, psn_sz; 434 u16 cmd_flags = 0, max_ssge; 435 u32 sw_prod, qp_flags = 0; 436 437 RCFW_CMD_PREP(req, CREATE_QP, cmd_flags); 438 439 /* General */ 440 req.type = qp->type; 441 req.dpi = cpu_to_le32(qp->dpi->dpi); 442 req.qp_handle = cpu_to_le64(qp->qp_handle); 443 444 /* SQ */ 445 psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ? 446 sizeof(struct sq_psn_search) : 0; 447 sq->hwq.max_elements = sq->max_wqe; 448 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist, 449 sq->nmap, &sq->hwq.max_elements, 450 BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 451 psn_sz, 452 PAGE_SIZE, HWQ_TYPE_QUEUE); 453 if (rc) 454 goto exit; 455 456 sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL); 457 if (!sq->swq) { 458 rc = -ENOMEM; 459 goto fail_sq; 460 } 461 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; 462 if (psn_sz) { 463 psn_search_ptr = (struct sq_psn_search **) 464 &hw_sq_send_ptr[get_sqe_pg 465 (sq->hwq.max_elements)]; 466 psn_search = (unsigned long int) 467 &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)] 468 [get_sqe_idx(sq->hwq.max_elements)]; 469 if (psn_search & ~PAGE_MASK) { 470 /* If the psn_search does not start on a page boundary, 471 * then calculate the offset 472 */ 473 poff = (psn_search & ~PAGE_MASK) / 474 BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE; 475 } 476 for (i = 0; i < sq->hwq.max_elements; i++) 477 sq->swq[i].psn_search = 478 &psn_search_ptr[get_psne_pg(i + poff)] 479 [get_psne_idx(i + poff)]; 480 } 481 pbl = &sq->hwq.pbl[PBL_LVL_0]; 482 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); 483 req.sq_pg_size_sq_lvl = 484 ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK) 485 << CMDQ_CREATE_QP_SQ_LVL_SFT) | 486 (pbl->pg_size == ROCE_PG_SIZE_4K ? 487 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K : 488 pbl->pg_size == ROCE_PG_SIZE_8K ? 489 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K : 490 pbl->pg_size == ROCE_PG_SIZE_64K ? 491 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K : 492 pbl->pg_size == ROCE_PG_SIZE_2M ? 493 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M : 494 pbl->pg_size == ROCE_PG_SIZE_8M ? 495 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M : 496 pbl->pg_size == ROCE_PG_SIZE_1G ? 497 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G : 498 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K); 499 500 /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */ 501 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; 502 for (sw_prod = 0; sw_prod < sq->hwq.max_elements; sw_prod++) { 503 hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)] 504 [get_sqe_idx(sw_prod)]; 505 hw_sq_send_hdr->wqe_type = SQ_BASE_WQE_TYPE_LOCAL_INVALID; 506 } 507 508 if (qp->scq) 509 req.scq_cid = cpu_to_le32(qp->scq->id); 510 511 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE; 512 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED; 513 if (qp->sig_type) 514 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION; 515 516 /* RQ */ 517 if (rq->max_wqe) { 518 rq->hwq.max_elements = rq->max_wqe; 519 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, rq->sglist, 520 rq->nmap, &rq->hwq.max_elements, 521 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0, 522 PAGE_SIZE, HWQ_TYPE_QUEUE); 523 if (rc) 524 goto fail_sq; 525 526 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq), 527 GFP_KERNEL); 528 if (!rq->swq) { 529 rc = -ENOMEM; 530 goto fail_rq; 531 } 532 pbl = &rq->hwq.pbl[PBL_LVL_0]; 533 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); 534 req.rq_pg_size_rq_lvl = 535 ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) << 536 CMDQ_CREATE_QP_RQ_LVL_SFT) | 537 (pbl->pg_size == ROCE_PG_SIZE_4K ? 538 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K : 539 pbl->pg_size == ROCE_PG_SIZE_8K ? 540 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K : 541 pbl->pg_size == ROCE_PG_SIZE_64K ? 542 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K : 543 pbl->pg_size == ROCE_PG_SIZE_2M ? 544 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M : 545 pbl->pg_size == ROCE_PG_SIZE_8M ? 546 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M : 547 pbl->pg_size == ROCE_PG_SIZE_1G ? 548 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G : 549 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K); 550 } 551 552 if (qp->rcq) 553 req.rcq_cid = cpu_to_le32(qp->rcq->id); 554 req.qp_flags = cpu_to_le32(qp_flags); 555 req.sq_size = cpu_to_le32(sq->hwq.max_elements); 556 req.rq_size = cpu_to_le32(rq->hwq.max_elements); 557 qp->sq_hdr_buf = NULL; 558 qp->rq_hdr_buf = NULL; 559 560 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); 561 if (rc) 562 goto fail_rq; 563 564 /* CTRL-22434: Irrespective of the requested SGE count on the SQ 565 * always create the QP with max send sges possible if the requested 566 * inline size is greater than 0. 567 */ 568 max_ssge = qp->max_inline_data ? 6 : sq->max_sge; 569 req.sq_fwo_sq_sge = cpu_to_le16( 570 ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK) 571 << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0); 572 req.rq_fwo_rq_sge = cpu_to_le16( 573 ((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK) 574 << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0); 575 /* ORRQ and IRRQ */ 576 if (psn_sz) { 577 xrrq = &qp->orrq; 578 xrrq->max_elements = 579 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); 580 req_size = xrrq->max_elements * 581 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1; 582 req_size &= ~(PAGE_SIZE - 1); 583 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0, 584 &xrrq->max_elements, 585 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE, 586 0, req_size, HWQ_TYPE_CTX); 587 if (rc) 588 goto fail_buf_free; 589 pbl = &xrrq->pbl[PBL_LVL_0]; 590 req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]); 591 592 xrrq = &qp->irrq; 593 xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS( 594 qp->max_dest_rd_atomic); 595 req_size = xrrq->max_elements * 596 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1; 597 req_size &= ~(PAGE_SIZE - 1); 598 599 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0, 600 &xrrq->max_elements, 601 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE, 602 0, req_size, HWQ_TYPE_CTX); 603 if (rc) 604 goto fail_orrq; 605 606 pbl = &xrrq->pbl[PBL_LVL_0]; 607 req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]); 608 } 609 req.pd_id = cpu_to_le32(qp->pd->id); 610 611 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 612 (void *)&resp, NULL, 0); 613 if (rc) 614 goto fail; 615 616 qp->id = le32_to_cpu(resp.xid); 617 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; 618 sq->flush_in_progress = false; 619 rq->flush_in_progress = false; 620 621 return 0; 622 623 fail: 624 if (qp->irrq.max_elements) 625 bnxt_qplib_free_hwq(res->pdev, &qp->irrq); 626 fail_orrq: 627 if (qp->orrq.max_elements) 628 bnxt_qplib_free_hwq(res->pdev, &qp->orrq); 629 fail_buf_free: 630 bnxt_qplib_free_qp_hdr_buf(res, qp); 631 fail_rq: 632 bnxt_qplib_free_hwq(res->pdev, &rq->hwq); 633 kfree(rq->swq); 634 fail_sq: 635 bnxt_qplib_free_hwq(res->pdev, &sq->hwq); 636 kfree(sq->swq); 637 exit: 638 return rc; 639 } 640 641 static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp) 642 { 643 switch (qp->state) { 644 case CMDQ_MODIFY_QP_NEW_STATE_RTR: 645 /* INIT->RTR, configure the path_mtu to the default 646 * 2048 if not being requested 647 */ 648 if (!(qp->modify_flags & 649 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) { 650 qp->modify_flags |= 651 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; 652 qp->path_mtu = 653 CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; 654 } 655 qp->modify_flags &= 656 ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; 657 /* Bono FW require the max_dest_rd_atomic to be >= 1 */ 658 if (qp->max_dest_rd_atomic < 1) 659 qp->max_dest_rd_atomic = 1; 660 qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC; 661 /* Bono FW 20.6.5 requires SGID_INDEX configuration */ 662 if (!(qp->modify_flags & 663 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) { 664 qp->modify_flags |= 665 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX; 666 qp->ah.sgid_index = 0; 667 } 668 break; 669 default: 670 break; 671 } 672 } 673 674 static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp *qp) 675 { 676 switch (qp->state) { 677 case CMDQ_MODIFY_QP_NEW_STATE_RTS: 678 /* Bono FW requires the max_rd_atomic to be >= 1 */ 679 if (qp->max_rd_atomic < 1) 680 qp->max_rd_atomic = 1; 681 /* Bono FW does not allow PKEY_INDEX, 682 * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT, 683 * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN, 684 * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID 685 * modification 686 */ 687 qp->modify_flags &= 688 ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY | 689 CMDQ_MODIFY_QP_MODIFY_MASK_DGID | 690 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | 691 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | 692 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | 693 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | 694 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | 695 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU | 696 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN | 697 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER | 698 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC | 699 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID); 700 break; 701 default: 702 break; 703 } 704 } 705 706 static void __filter_modify_flags(struct bnxt_qplib_qp *qp) 707 { 708 switch (qp->cur_qp_state) { 709 case CMDQ_MODIFY_QP_NEW_STATE_RESET: 710 break; 711 case CMDQ_MODIFY_QP_NEW_STATE_INIT: 712 __modify_flags_from_init_state(qp); 713 break; 714 case CMDQ_MODIFY_QP_NEW_STATE_RTR: 715 __modify_flags_from_rtr_state(qp); 716 break; 717 case CMDQ_MODIFY_QP_NEW_STATE_RTS: 718 break; 719 case CMDQ_MODIFY_QP_NEW_STATE_SQD: 720 break; 721 case CMDQ_MODIFY_QP_NEW_STATE_SQE: 722 break; 723 case CMDQ_MODIFY_QP_NEW_STATE_ERR: 724 break; 725 default: 726 break; 727 } 728 } 729 730 int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) 731 { 732 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 733 struct cmdq_modify_qp req; 734 struct creq_modify_qp_resp resp; 735 u16 cmd_flags = 0, pkey; 736 u32 temp32[4]; 737 u32 bmask; 738 int rc; 739 740 RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags); 741 742 /* Filter out the qp_attr_mask based on the state->new transition */ 743 __filter_modify_flags(qp); 744 bmask = qp->modify_flags; 745 req.modify_mask = cpu_to_le32(qp->modify_flags); 746 req.qp_cid = cpu_to_le32(qp->id); 747 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) { 748 req.network_type_en_sqd_async_notify_new_state = 749 (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) | 750 (qp->en_sqd_async_notify ? 751 CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0); 752 } 753 req.network_type_en_sqd_async_notify_new_state |= qp->nw_type; 754 755 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS) 756 req.access = qp->access; 757 758 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) { 759 if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl, 760 qp->pkey_index, &pkey)) 761 req.pkey = cpu_to_le16(pkey); 762 } 763 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY) 764 req.qkey = cpu_to_le32(qp->qkey); 765 766 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) { 767 memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid)); 768 req.dgid[0] = cpu_to_le32(temp32[0]); 769 req.dgid[1] = cpu_to_le32(temp32[1]); 770 req.dgid[2] = cpu_to_le32(temp32[2]); 771 req.dgid[3] = cpu_to_le32(temp32[3]); 772 } 773 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL) 774 req.flow_label = cpu_to_le32(qp->ah.flow_label); 775 776 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX) 777 req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id 778 [qp->ah.sgid_index]); 779 780 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT) 781 req.hop_limit = qp->ah.hop_limit; 782 783 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS) 784 req.traffic_class = qp->ah.traffic_class; 785 786 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC) 787 memcpy(req.dest_mac, qp->ah.dmac, 6); 788 789 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU) 790 req.path_mtu = qp->path_mtu; 791 792 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT) 793 req.timeout = qp->timeout; 794 795 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT) 796 req.retry_cnt = qp->retry_cnt; 797 798 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY) 799 req.rnr_retry = qp->rnr_retry; 800 801 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER) 802 req.min_rnr_timer = qp->min_rnr_timer; 803 804 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN) 805 req.rq_psn = cpu_to_le32(qp->rq.psn); 806 807 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN) 808 req.sq_psn = cpu_to_le32(qp->sq.psn); 809 810 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC) 811 req.max_rd_atomic = 812 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); 813 814 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC) 815 req.max_dest_rd_atomic = 816 IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic); 817 818 req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements); 819 req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements); 820 req.sq_sge = cpu_to_le16(qp->sq.max_sge); 821 req.rq_sge = cpu_to_le16(qp->rq.max_sge); 822 req.max_inline_data = cpu_to_le32(qp->max_inline_data); 823 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID) 824 req.dest_qp_id = cpu_to_le32(qp->dest_qpn); 825 826 req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id); 827 828 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 829 (void *)&resp, NULL, 0); 830 if (rc) 831 return rc; 832 qp->cur_qp_state = qp->state; 833 return 0; 834 } 835 836 int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) 837 { 838 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 839 struct cmdq_query_qp req; 840 struct creq_query_qp_resp resp; 841 struct bnxt_qplib_rcfw_sbuf *sbuf; 842 struct creq_query_qp_resp_sb *sb; 843 u16 cmd_flags = 0; 844 u32 temp32[4]; 845 int i, rc = 0; 846 847 RCFW_CMD_PREP(req, QUERY_QP, cmd_flags); 848 849 sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb)); 850 if (!sbuf) 851 return -ENOMEM; 852 sb = sbuf->sb; 853 854 req.qp_cid = cpu_to_le32(qp->id); 855 req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS; 856 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, 857 (void *)sbuf, 0); 858 if (rc) 859 goto bail; 860 /* Extract the context from the side buffer */ 861 qp->state = sb->en_sqd_async_notify_state & 862 CREQ_QUERY_QP_RESP_SB_STATE_MASK; 863 qp->en_sqd_async_notify = sb->en_sqd_async_notify_state & 864 CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ? 865 true : false; 866 qp->access = sb->access; 867 qp->pkey_index = le16_to_cpu(sb->pkey); 868 qp->qkey = le32_to_cpu(sb->qkey); 869 870 temp32[0] = le32_to_cpu(sb->dgid[0]); 871 temp32[1] = le32_to_cpu(sb->dgid[1]); 872 temp32[2] = le32_to_cpu(sb->dgid[2]); 873 temp32[3] = le32_to_cpu(sb->dgid[3]); 874 memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data)); 875 876 qp->ah.flow_label = le32_to_cpu(sb->flow_label); 877 878 qp->ah.sgid_index = 0; 879 for (i = 0; i < res->sgid_tbl.max; i++) { 880 if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) { 881 qp->ah.sgid_index = i; 882 break; 883 } 884 } 885 if (i == res->sgid_tbl.max) 886 dev_warn(&res->pdev->dev, "QPLIB: SGID not found??"); 887 888 qp->ah.hop_limit = sb->hop_limit; 889 qp->ah.traffic_class = sb->traffic_class; 890 memcpy(qp->ah.dmac, sb->dest_mac, 6); 891 qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) & 892 CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK) >> 893 CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT; 894 qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) & 895 CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) >> 896 CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT; 897 qp->timeout = sb->timeout; 898 qp->retry_cnt = sb->retry_cnt; 899 qp->rnr_retry = sb->rnr_retry; 900 qp->min_rnr_timer = sb->min_rnr_timer; 901 qp->rq.psn = le32_to_cpu(sb->rq_psn); 902 qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic); 903 qp->sq.psn = le32_to_cpu(sb->sq_psn); 904 qp->max_dest_rd_atomic = 905 IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic); 906 qp->sq.max_wqe = qp->sq.hwq.max_elements; 907 qp->rq.max_wqe = qp->rq.hwq.max_elements; 908 qp->sq.max_sge = le16_to_cpu(sb->sq_sge); 909 qp->rq.max_sge = le16_to_cpu(sb->rq_sge); 910 qp->max_inline_data = le32_to_cpu(sb->max_inline_data); 911 qp->dest_qpn = le32_to_cpu(sb->dest_qp_id); 912 memcpy(qp->smac, sb->src_mac, 6); 913 qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id); 914 bail: 915 bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf); 916 return rc; 917 } 918 919 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp) 920 { 921 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; 922 struct cq_base *hw_cqe, **hw_cqe_ptr; 923 int i; 924 925 for (i = 0; i < cq_hwq->max_elements; i++) { 926 hw_cqe_ptr = (struct cq_base **)cq_hwq->pbl_ptr; 927 hw_cqe = &hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)]; 928 if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements)) 929 continue; 930 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { 931 case CQ_BASE_CQE_TYPE_REQ: 932 case CQ_BASE_CQE_TYPE_TERMINAL: 933 { 934 struct cq_req *cqe = (struct cq_req *)hw_cqe; 935 936 if (qp == le64_to_cpu(cqe->qp_handle)) 937 cqe->qp_handle = 0; 938 break; 939 } 940 case CQ_BASE_CQE_TYPE_RES_RC: 941 case CQ_BASE_CQE_TYPE_RES_UD: 942 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: 943 { 944 struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe; 945 946 if (qp == le64_to_cpu(cqe->qp_handle)) 947 cqe->qp_handle = 0; 948 break; 949 } 950 default: 951 break; 952 } 953 } 954 } 955 956 int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, 957 struct bnxt_qplib_qp *qp) 958 { 959 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 960 struct cmdq_destroy_qp req; 961 struct creq_destroy_qp_resp resp; 962 unsigned long flags; 963 u16 cmd_flags = 0; 964 int rc; 965 966 RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags); 967 968 req.qp_cid = cpu_to_le32(qp->id); 969 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 970 (void *)&resp, NULL, 0); 971 if (rc) 972 return rc; 973 974 /* Must walk the associated CQs to nullified the QP ptr */ 975 spin_lock_irqsave(&qp->scq->hwq.lock, flags); 976 977 __clean_cq(qp->scq, (u64)(unsigned long)qp); 978 979 if (qp->rcq && qp->rcq != qp->scq) { 980 spin_lock(&qp->rcq->hwq.lock); 981 __clean_cq(qp->rcq, (u64)(unsigned long)qp); 982 spin_unlock(&qp->rcq->hwq.lock); 983 } 984 985 spin_unlock_irqrestore(&qp->scq->hwq.lock, flags); 986 987 bnxt_qplib_free_qp_hdr_buf(res, qp); 988 bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq); 989 kfree(qp->sq.swq); 990 991 bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq); 992 kfree(qp->rq.swq); 993 994 if (qp->irrq.max_elements) 995 bnxt_qplib_free_hwq(res->pdev, &qp->irrq); 996 if (qp->orrq.max_elements) 997 bnxt_qplib_free_hwq(res->pdev, &qp->orrq); 998 999 return 0; 1000 } 1001 1002 void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, 1003 struct bnxt_qplib_sge *sge) 1004 { 1005 struct bnxt_qplib_q *sq = &qp->sq; 1006 u32 sw_prod; 1007 1008 memset(sge, 0, sizeof(*sge)); 1009 1010 if (qp->sq_hdr_buf) { 1011 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); 1012 sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map + 1013 sw_prod * qp->sq_hdr_buf_size); 1014 sge->lkey = 0xFFFFFFFF; 1015 sge->size = qp->sq_hdr_buf_size; 1016 return qp->sq_hdr_buf + sw_prod * sge->size; 1017 } 1018 return NULL; 1019 } 1020 1021 u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp) 1022 { 1023 struct bnxt_qplib_q *rq = &qp->rq; 1024 1025 return HWQ_CMP(rq->hwq.prod, &rq->hwq); 1026 } 1027 1028 dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index) 1029 { 1030 return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size); 1031 } 1032 1033 void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, 1034 struct bnxt_qplib_sge *sge) 1035 { 1036 struct bnxt_qplib_q *rq = &qp->rq; 1037 u32 sw_prod; 1038 1039 memset(sge, 0, sizeof(*sge)); 1040 1041 if (qp->rq_hdr_buf) { 1042 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); 1043 sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map + 1044 sw_prod * qp->rq_hdr_buf_size); 1045 sge->lkey = 0xFFFFFFFF; 1046 sge->size = qp->rq_hdr_buf_size; 1047 return qp->rq_hdr_buf + sw_prod * sge->size; 1048 } 1049 return NULL; 1050 } 1051 1052 void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp) 1053 { 1054 struct bnxt_qplib_q *sq = &qp->sq; 1055 struct dbr_dbr db_msg = { 0 }; 1056 u32 sw_prod; 1057 1058 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); 1059 1060 db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) & 1061 DBR_DBR_INDEX_MASK); 1062 db_msg.type_xid = 1063 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | 1064 DBR_DBR_TYPE_SQ); 1065 /* Flush all the WQE writes to HW */ 1066 wmb(); 1067 __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); 1068 } 1069 1070 int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, 1071 struct bnxt_qplib_swqe *wqe) 1072 { 1073 struct bnxt_qplib_q *sq = &qp->sq; 1074 struct bnxt_qplib_swq *swq; 1075 struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr; 1076 struct sq_sge *hw_sge; 1077 u32 sw_prod; 1078 u8 wqe_size16; 1079 int i, rc = 0, data_len = 0, pkt_num = 0; 1080 __le32 temp32; 1081 1082 if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) { 1083 rc = -EINVAL; 1084 goto done; 1085 } 1086 1087 if (bnxt_qplib_queue_full(sq)) { 1088 dev_err(&sq->hwq.pdev->dev, 1089 "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x", 1090 sq->hwq.prod, sq->hwq.cons, sq->hwq.max_elements, 1091 sq->q_full_delta); 1092 rc = -ENOMEM; 1093 goto done; 1094 } 1095 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); 1096 swq = &sq->swq[sw_prod]; 1097 swq->wr_id = wqe->wr_id; 1098 swq->type = wqe->type; 1099 swq->flags = wqe->flags; 1100 if (qp->sig_type) 1101 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP; 1102 swq->start_psn = sq->psn & BTH_PSN_MASK; 1103 1104 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; 1105 hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)] 1106 [get_sqe_idx(sw_prod)]; 1107 1108 memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE); 1109 1110 if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) { 1111 /* Copy the inline data */ 1112 if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { 1113 dev_warn(&sq->hwq.pdev->dev, 1114 "QPLIB: Inline data length > 96 detected"); 1115 data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH; 1116 } else { 1117 data_len = wqe->inline_len; 1118 } 1119 memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len); 1120 wqe_size16 = (data_len + 15) >> 4; 1121 } else { 1122 for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data; 1123 i < wqe->num_sge; i++, hw_sge++) { 1124 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); 1125 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); 1126 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); 1127 data_len += wqe->sg_list[i].size; 1128 } 1129 /* Each SGE entry = 1 WQE size16 */ 1130 wqe_size16 = wqe->num_sge; 1131 } 1132 1133 /* Specifics */ 1134 switch (wqe->type) { 1135 case BNXT_QPLIB_SWQE_TYPE_SEND: 1136 if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) { 1137 /* Assemble info for Raw Ethertype QPs */ 1138 struct sq_send_raweth_qp1 *sqe = 1139 (struct sq_send_raweth_qp1 *)hw_sq_send_hdr; 1140 1141 sqe->wqe_type = wqe->type; 1142 sqe->flags = wqe->flags; 1143 sqe->wqe_size = wqe_size16 + 1144 ((offsetof(typeof(*sqe), data) + 15) >> 4); 1145 sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action); 1146 sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags); 1147 sqe->length = cpu_to_le32(data_len); 1148 sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta & 1149 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) << 1150 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT); 1151 1152 break; 1153 } 1154 /* else, just fall thru */ 1155 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: 1156 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: 1157 { 1158 struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr; 1159 1160 sqe->wqe_type = wqe->type; 1161 sqe->flags = wqe->flags; 1162 sqe->wqe_size = wqe_size16 + 1163 ((offsetof(typeof(*sqe), data) + 15) >> 4); 1164 sqe->inv_key_or_imm_data = cpu_to_le32( 1165 wqe->send.inv_key); 1166 if (qp->type == CMDQ_CREATE_QP_TYPE_UD) { 1167 sqe->q_key = cpu_to_le32(wqe->send.q_key); 1168 sqe->dst_qp = cpu_to_le32( 1169 wqe->send.dst_qp & SQ_SEND_DST_QP_MASK); 1170 sqe->length = cpu_to_le32(data_len); 1171 sqe->avid = cpu_to_le32(wqe->send.avid & 1172 SQ_SEND_AVID_MASK); 1173 sq->psn = (sq->psn + 1) & BTH_PSN_MASK; 1174 } else { 1175 sqe->length = cpu_to_le32(data_len); 1176 sqe->dst_qp = 0; 1177 sqe->avid = 0; 1178 if (qp->mtu) 1179 pkt_num = (data_len + qp->mtu - 1) / qp->mtu; 1180 if (!pkt_num) 1181 pkt_num = 1; 1182 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; 1183 } 1184 break; 1185 } 1186 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: 1187 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: 1188 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: 1189 { 1190 struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr; 1191 1192 sqe->wqe_type = wqe->type; 1193 sqe->flags = wqe->flags; 1194 sqe->wqe_size = wqe_size16 + 1195 ((offsetof(typeof(*sqe), data) + 15) >> 4); 1196 sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key); 1197 sqe->length = cpu_to_le32((u32)data_len); 1198 sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va); 1199 sqe->remote_key = cpu_to_le32(wqe->rdma.r_key); 1200 if (qp->mtu) 1201 pkt_num = (data_len + qp->mtu - 1) / qp->mtu; 1202 if (!pkt_num) 1203 pkt_num = 1; 1204 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; 1205 break; 1206 } 1207 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: 1208 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: 1209 { 1210 struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr; 1211 1212 sqe->wqe_type = wqe->type; 1213 sqe->flags = wqe->flags; 1214 sqe->remote_key = cpu_to_le32(wqe->atomic.r_key); 1215 sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va); 1216 sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data); 1217 sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data); 1218 if (qp->mtu) 1219 pkt_num = (data_len + qp->mtu - 1) / qp->mtu; 1220 if (!pkt_num) 1221 pkt_num = 1; 1222 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; 1223 break; 1224 } 1225 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: 1226 { 1227 struct sq_localinvalidate *sqe = 1228 (struct sq_localinvalidate *)hw_sq_send_hdr; 1229 1230 sqe->wqe_type = wqe->type; 1231 sqe->flags = wqe->flags; 1232 sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key); 1233 1234 break; 1235 } 1236 case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR: 1237 { 1238 struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr; 1239 1240 sqe->wqe_type = wqe->type; 1241 sqe->flags = wqe->flags; 1242 sqe->access_cntl = wqe->frmr.access_cntl | 1243 SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; 1244 sqe->zero_based_page_size_log = 1245 (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) << 1246 SQ_FR_PMR_PAGE_SIZE_LOG_SFT | 1247 (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0); 1248 sqe->l_key = cpu_to_le32(wqe->frmr.l_key); 1249 temp32 = cpu_to_le32(wqe->frmr.length); 1250 memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length)); 1251 sqe->numlevels_pbl_page_size_log = 1252 ((wqe->frmr.pbl_pg_sz_log << 1253 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) & 1254 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) | 1255 ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) & 1256 SQ_FR_PMR_NUMLEVELS_MASK); 1257 1258 for (i = 0; i < wqe->frmr.page_list_len; i++) 1259 wqe->frmr.pbl_ptr[i] = cpu_to_le64( 1260 wqe->frmr.page_list[i] | 1261 PTU_PTE_VALID); 1262 sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr); 1263 sqe->va = cpu_to_le64(wqe->frmr.va); 1264 1265 break; 1266 } 1267 case BNXT_QPLIB_SWQE_TYPE_BIND_MW: 1268 { 1269 struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr; 1270 1271 sqe->wqe_type = wqe->type; 1272 sqe->flags = wqe->flags; 1273 sqe->access_cntl = wqe->bind.access_cntl; 1274 sqe->mw_type_zero_based = wqe->bind.mw_type | 1275 (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0); 1276 sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key); 1277 sqe->l_key = cpu_to_le32(wqe->bind.r_key); 1278 sqe->va = cpu_to_le64(wqe->bind.va); 1279 temp32 = cpu_to_le32(wqe->bind.length); 1280 memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length)); 1281 break; 1282 } 1283 default: 1284 /* Bad wqe, return error */ 1285 rc = -EINVAL; 1286 goto done; 1287 } 1288 swq->next_psn = sq->psn & BTH_PSN_MASK; 1289 if (swq->psn_search) { 1290 swq->psn_search->opcode_start_psn = cpu_to_le32( 1291 ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) & 1292 SQ_PSN_SEARCH_START_PSN_MASK) | 1293 ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) & 1294 SQ_PSN_SEARCH_OPCODE_MASK)); 1295 swq->psn_search->flags_next_psn = cpu_to_le32( 1296 ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) & 1297 SQ_PSN_SEARCH_NEXT_PSN_MASK)); 1298 } 1299 1300 sq->hwq.prod++; 1301 1302 qp->wqe_cnt++; 1303 1304 done: 1305 return rc; 1306 } 1307 1308 void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp) 1309 { 1310 struct bnxt_qplib_q *rq = &qp->rq; 1311 struct dbr_dbr db_msg = { 0 }; 1312 u32 sw_prod; 1313 1314 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); 1315 db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) & 1316 DBR_DBR_INDEX_MASK); 1317 db_msg.type_xid = 1318 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | 1319 DBR_DBR_TYPE_RQ); 1320 1321 /* Flush the writes to HW Rx WQE before the ringing Rx DB */ 1322 wmb(); 1323 __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); 1324 } 1325 1326 int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, 1327 struct bnxt_qplib_swqe *wqe) 1328 { 1329 struct bnxt_qplib_q *rq = &qp->rq; 1330 struct rq_wqe *rqe, **rqe_ptr; 1331 struct sq_sge *hw_sge; 1332 u32 sw_prod; 1333 int i, rc = 0; 1334 1335 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { 1336 dev_err(&rq->hwq.pdev->dev, 1337 "QPLIB: FP: QP (0x%x) is in the 0x%x state", 1338 qp->id, qp->state); 1339 rc = -EINVAL; 1340 goto done; 1341 } 1342 if (bnxt_qplib_queue_full(rq)) { 1343 dev_err(&rq->hwq.pdev->dev, 1344 "QPLIB: FP: QP (0x%x) RQ is full!", qp->id); 1345 rc = -EINVAL; 1346 goto done; 1347 } 1348 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); 1349 rq->swq[sw_prod].wr_id = wqe->wr_id; 1350 1351 rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr; 1352 rqe = &rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)]; 1353 1354 memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE); 1355 1356 /* Calculate wqe_size16 and data_len */ 1357 for (i = 0, hw_sge = (struct sq_sge *)rqe->data; 1358 i < wqe->num_sge; i++, hw_sge++) { 1359 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); 1360 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); 1361 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); 1362 } 1363 rqe->wqe_type = wqe->type; 1364 rqe->flags = wqe->flags; 1365 rqe->wqe_size = wqe->num_sge + 1366 ((offsetof(typeof(*rqe), data) + 15) >> 4); 1367 1368 /* Supply the rqe->wr_id index to the wr_id_tbl for now */ 1369 rqe->wr_id[0] = cpu_to_le32(sw_prod); 1370 1371 rq->hwq.prod++; 1372 done: 1373 return rc; 1374 } 1375 1376 /* CQ */ 1377 1378 /* Spinlock must be held */ 1379 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq) 1380 { 1381 struct dbr_dbr db_msg = { 0 }; 1382 1383 db_msg.type_xid = 1384 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | 1385 DBR_DBR_TYPE_CQ_ARMENA); 1386 /* Flush memory writes before enabling the CQ */ 1387 wmb(); 1388 __iowrite64_copy(cq->dbr_base, &db_msg, sizeof(db_msg) / sizeof(u64)); 1389 } 1390 1391 static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type) 1392 { 1393 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; 1394 struct dbr_dbr db_msg = { 0 }; 1395 u32 sw_cons; 1396 1397 /* Ring DB */ 1398 sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq); 1399 db_msg.index = cpu_to_le32((sw_cons << DBR_DBR_INDEX_SFT) & 1400 DBR_DBR_INDEX_MASK); 1401 db_msg.type_xid = 1402 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | 1403 arm_type); 1404 /* flush memory writes before arming the CQ */ 1405 wmb(); 1406 __iowrite64_copy(cq->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); 1407 } 1408 1409 int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) 1410 { 1411 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 1412 struct cmdq_create_cq req; 1413 struct creq_create_cq_resp resp; 1414 struct bnxt_qplib_pbl *pbl; 1415 u16 cmd_flags = 0; 1416 int rc; 1417 1418 cq->hwq.max_elements = cq->max_wqe; 1419 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, cq->sghead, 1420 cq->nmap, &cq->hwq.max_elements, 1421 BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0, 1422 PAGE_SIZE, HWQ_TYPE_QUEUE); 1423 if (rc) 1424 goto exit; 1425 1426 RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags); 1427 1428 if (!cq->dpi) { 1429 dev_err(&rcfw->pdev->dev, 1430 "QPLIB: FP: CREATE_CQ failed due to NULL DPI"); 1431 return -EINVAL; 1432 } 1433 req.dpi = cpu_to_le32(cq->dpi->dpi); 1434 req.cq_handle = cpu_to_le64(cq->cq_handle); 1435 1436 req.cq_size = cpu_to_le32(cq->hwq.max_elements); 1437 pbl = &cq->hwq.pbl[PBL_LVL_0]; 1438 req.pg_size_lvl = cpu_to_le32( 1439 ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) << 1440 CMDQ_CREATE_CQ_LVL_SFT) | 1441 (pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K : 1442 pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K : 1443 pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K : 1444 pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M : 1445 pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M : 1446 pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G : 1447 CMDQ_CREATE_CQ_PG_SIZE_PG_4K)); 1448 1449 req.pbl = cpu_to_le64(pbl->pg_map_arr[0]); 1450 1451 req.cq_fco_cnq_id = cpu_to_le32( 1452 (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) << 1453 CMDQ_CREATE_CQ_CNQ_ID_SFT); 1454 1455 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 1456 (void *)&resp, NULL, 0); 1457 if (rc) 1458 goto fail; 1459 1460 cq->id = le32_to_cpu(resp.xid); 1461 cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem; 1462 cq->period = BNXT_QPLIB_QUEUE_START_PERIOD; 1463 init_waitqueue_head(&cq->waitq); 1464 1465 bnxt_qplib_arm_cq_enable(cq); 1466 return 0; 1467 1468 fail: 1469 bnxt_qplib_free_hwq(res->pdev, &cq->hwq); 1470 exit: 1471 return rc; 1472 } 1473 1474 int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) 1475 { 1476 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 1477 struct cmdq_destroy_cq req; 1478 struct creq_destroy_cq_resp resp; 1479 u16 cmd_flags = 0; 1480 int rc; 1481 1482 RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags); 1483 1484 req.cq_cid = cpu_to_le32(cq->id); 1485 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 1486 (void *)&resp, NULL, 0); 1487 if (rc) 1488 return rc; 1489 bnxt_qplib_free_hwq(res->pdev, &cq->hwq); 1490 return 0; 1491 } 1492 1493 static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp, 1494 struct bnxt_qplib_cqe **pcqe, int *budget) 1495 { 1496 u32 sw_prod, sw_cons; 1497 struct bnxt_qplib_cqe *cqe; 1498 int rc = 0; 1499 1500 /* Now complete all outstanding SQEs with FLUSHED_ERR */ 1501 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); 1502 cqe = *pcqe; 1503 while (*budget) { 1504 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); 1505 if (sw_cons == sw_prod) { 1506 sq->flush_in_progress = false; 1507 break; 1508 } 1509 memset(cqe, 0, sizeof(*cqe)); 1510 cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR; 1511 cqe->opcode = CQ_BASE_CQE_TYPE_REQ; 1512 cqe->qp_handle = (u64)(unsigned long)qp; 1513 cqe->wr_id = sq->swq[sw_cons].wr_id; 1514 cqe->src_qp = qp->id; 1515 cqe->type = sq->swq[sw_cons].type; 1516 cqe++; 1517 (*budget)--; 1518 sq->hwq.cons++; 1519 } 1520 *pcqe = cqe; 1521 if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod) 1522 /* Out of budget */ 1523 rc = -EAGAIN; 1524 1525 return rc; 1526 } 1527 1528 static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp, 1529 int opcode, struct bnxt_qplib_cqe **pcqe, int *budget) 1530 { 1531 struct bnxt_qplib_cqe *cqe; 1532 u32 sw_prod, sw_cons; 1533 int rc = 0; 1534 1535 /* Flush the rest of the RQ */ 1536 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); 1537 cqe = *pcqe; 1538 while (*budget) { 1539 sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq); 1540 if (sw_cons == sw_prod) 1541 break; 1542 memset(cqe, 0, sizeof(*cqe)); 1543 cqe->status = 1544 CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR; 1545 cqe->opcode = opcode; 1546 cqe->qp_handle = (unsigned long)qp; 1547 cqe->wr_id = rq->swq[sw_cons].wr_id; 1548 cqe++; 1549 (*budget)--; 1550 rq->hwq.cons++; 1551 } 1552 *pcqe = cqe; 1553 if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod) 1554 /* Out of budget */ 1555 rc = -EAGAIN; 1556 1557 return rc; 1558 } 1559 1560 /* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive) 1561 * CQE is track from sw_cq_cons to max_element but valid only if VALID=1 1562 */ 1563 static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq, 1564 u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons) 1565 { 1566 struct bnxt_qplib_q *sq = &qp->sq; 1567 struct bnxt_qplib_swq *swq; 1568 u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx; 1569 struct cq_base *peek_hwcqe, **peek_hw_cqe_ptr; 1570 struct cq_req *peek_req_hwcqe; 1571 struct bnxt_qplib_qp *peek_qp; 1572 struct bnxt_qplib_q *peek_sq; 1573 int i, rc = 0; 1574 1575 /* Normal mode */ 1576 /* Check for the psn_search marking before completing */ 1577 swq = &sq->swq[sw_sq_cons]; 1578 if (swq->psn_search && 1579 le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) { 1580 /* Unmark */ 1581 swq->psn_search->flags_next_psn = cpu_to_le32 1582 (le32_to_cpu(swq->psn_search->flags_next_psn) 1583 & ~0x80000000); 1584 dev_dbg(&cq->hwq.pdev->dev, 1585 "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n", 1586 cq_cons, qp->id, sw_sq_cons, cqe_sq_cons); 1587 sq->condition = true; 1588 sq->send_phantom = true; 1589 1590 /* TODO: Only ARM if the previous SQE is ARMALL */ 1591 bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ_ARMALL); 1592 1593 rc = -EAGAIN; 1594 goto out; 1595 } 1596 if (sq->condition) { 1597 /* Peek at the completions */ 1598 peek_raw_cq_cons = cq->hwq.cons; 1599 peek_sw_cq_cons = cq_cons; 1600 i = cq->hwq.max_elements; 1601 while (i--) { 1602 peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq); 1603 peek_hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr; 1604 peek_hwcqe = &peek_hw_cqe_ptr[CQE_PG(peek_sw_cq_cons)] 1605 [CQE_IDX(peek_sw_cq_cons)]; 1606 /* If the next hwcqe is VALID */ 1607 if (CQE_CMP_VALID(peek_hwcqe, peek_raw_cq_cons, 1608 cq->hwq.max_elements)) { 1609 /* If the next hwcqe is a REQ */ 1610 if ((peek_hwcqe->cqe_type_toggle & 1611 CQ_BASE_CQE_TYPE_MASK) == 1612 CQ_BASE_CQE_TYPE_REQ) { 1613 peek_req_hwcqe = (struct cq_req *) 1614 peek_hwcqe; 1615 peek_qp = (struct bnxt_qplib_qp *) 1616 ((unsigned long) 1617 le64_to_cpu 1618 (peek_req_hwcqe->qp_handle)); 1619 peek_sq = &peek_qp->sq; 1620 peek_sq_cons_idx = HWQ_CMP(le16_to_cpu( 1621 peek_req_hwcqe->sq_cons_idx) - 1 1622 , &sq->hwq); 1623 /* If the hwcqe's sq's wr_id matches */ 1624 if (peek_sq == sq && 1625 sq->swq[peek_sq_cons_idx].wr_id == 1626 BNXT_QPLIB_FENCE_WRID) { 1627 /* 1628 * Unbreak only if the phantom 1629 * comes back 1630 */ 1631 dev_dbg(&cq->hwq.pdev->dev, 1632 "FP:Got Phantom CQE"); 1633 sq->condition = false; 1634 sq->single = true; 1635 rc = 0; 1636 goto out; 1637 } 1638 } 1639 /* Valid but not the phantom, so keep looping */ 1640 } else { 1641 /* Not valid yet, just exit and wait */ 1642 rc = -EINVAL; 1643 goto out; 1644 } 1645 peek_sw_cq_cons++; 1646 peek_raw_cq_cons++; 1647 } 1648 dev_err(&cq->hwq.pdev->dev, 1649 "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x", 1650 cq_cons, qp->id, sw_sq_cons, cqe_sq_cons); 1651 rc = -EINVAL; 1652 } 1653 out: 1654 return rc; 1655 } 1656 1657 static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, 1658 struct cq_req *hwcqe, 1659 struct bnxt_qplib_cqe **pcqe, int *budget, 1660 u32 cq_cons, struct bnxt_qplib_qp **lib_qp) 1661 { 1662 struct bnxt_qplib_qp *qp; 1663 struct bnxt_qplib_q *sq; 1664 struct bnxt_qplib_cqe *cqe; 1665 u32 sw_sq_cons, cqe_sq_cons; 1666 struct bnxt_qplib_swq *swq; 1667 int rc = 0; 1668 1669 qp = (struct bnxt_qplib_qp *)((unsigned long) 1670 le64_to_cpu(hwcqe->qp_handle)); 1671 if (!qp) { 1672 dev_err(&cq->hwq.pdev->dev, 1673 "QPLIB: FP: Process Req qp is NULL"); 1674 return -EINVAL; 1675 } 1676 sq = &qp->sq; 1677 1678 cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq); 1679 if (cqe_sq_cons > sq->hwq.max_elements) { 1680 dev_err(&cq->hwq.pdev->dev, 1681 "QPLIB: FP: CQ Process req reported "); 1682 dev_err(&cq->hwq.pdev->dev, 1683 "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x", 1684 cqe_sq_cons, sq->hwq.max_elements); 1685 return -EINVAL; 1686 } 1687 /* If we were in the middle of flushing the SQ, continue */ 1688 if (sq->flush_in_progress) 1689 goto flush; 1690 1691 /* Require to walk the sq's swq to fabricate CQEs for all previously 1692 * signaled SWQEs due to CQE aggregation from the current sq cons 1693 * to the cqe_sq_cons 1694 */ 1695 cqe = *pcqe; 1696 while (*budget) { 1697 sw_sq_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); 1698 if (sw_sq_cons == cqe_sq_cons) 1699 /* Done */ 1700 break; 1701 1702 swq = &sq->swq[sw_sq_cons]; 1703 memset(cqe, 0, sizeof(*cqe)); 1704 cqe->opcode = CQ_BASE_CQE_TYPE_REQ; 1705 cqe->qp_handle = (u64)(unsigned long)qp; 1706 cqe->src_qp = qp->id; 1707 cqe->wr_id = swq->wr_id; 1708 if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID) 1709 goto skip; 1710 cqe->type = swq->type; 1711 1712 /* For the last CQE, check for status. For errors, regardless 1713 * of the request being signaled or not, it must complete with 1714 * the hwcqe error status 1715 */ 1716 if (HWQ_CMP((sw_sq_cons + 1), &sq->hwq) == cqe_sq_cons && 1717 hwcqe->status != CQ_REQ_STATUS_OK) { 1718 cqe->status = hwcqe->status; 1719 dev_err(&cq->hwq.pdev->dev, 1720 "QPLIB: FP: CQ Processed Req "); 1721 dev_err(&cq->hwq.pdev->dev, 1722 "QPLIB: wr_id[%d] = 0x%llx with status 0x%x", 1723 sw_sq_cons, cqe->wr_id, cqe->status); 1724 cqe++; 1725 (*budget)--; 1726 sq->flush_in_progress = true; 1727 /* Must block new posting of SQ and RQ */ 1728 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; 1729 sq->condition = false; 1730 sq->single = false; 1731 } else { 1732 if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) { 1733 /* Before we complete, do WA 9060 */ 1734 if (do_wa9060(qp, cq, cq_cons, sw_sq_cons, 1735 cqe_sq_cons)) { 1736 *lib_qp = qp; 1737 goto out; 1738 } 1739 cqe->status = CQ_REQ_STATUS_OK; 1740 cqe++; 1741 (*budget)--; 1742 } 1743 } 1744 skip: 1745 sq->hwq.cons++; 1746 if (sq->single) 1747 break; 1748 } 1749 out: 1750 *pcqe = cqe; 1751 if (HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_sq_cons) { 1752 /* Out of budget */ 1753 rc = -EAGAIN; 1754 goto done; 1755 } 1756 /* 1757 * Back to normal completion mode only after it has completed all of 1758 * the WC for this CQE 1759 */ 1760 sq->single = false; 1761 if (!sq->flush_in_progress) 1762 goto done; 1763 flush: 1764 /* Require to walk the sq's swq to fabricate CQEs for all 1765 * previously posted SWQEs due to the error CQE received 1766 */ 1767 rc = __flush_sq(sq, qp, pcqe, budget); 1768 if (!rc) 1769 sq->flush_in_progress = false; 1770 done: 1771 return rc; 1772 } 1773 1774 static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq, 1775 struct cq_res_rc *hwcqe, 1776 struct bnxt_qplib_cqe **pcqe, 1777 int *budget) 1778 { 1779 struct bnxt_qplib_qp *qp; 1780 struct bnxt_qplib_q *rq; 1781 struct bnxt_qplib_cqe *cqe; 1782 u32 wr_id_idx; 1783 int rc = 0; 1784 1785 qp = (struct bnxt_qplib_qp *)((unsigned long) 1786 le64_to_cpu(hwcqe->qp_handle)); 1787 if (!qp) { 1788 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL"); 1789 return -EINVAL; 1790 } 1791 cqe = *pcqe; 1792 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; 1793 cqe->length = le32_to_cpu(hwcqe->length); 1794 cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key); 1795 cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle); 1796 cqe->flags = le16_to_cpu(hwcqe->flags); 1797 cqe->status = hwcqe->status; 1798 cqe->qp_handle = (u64)(unsigned long)qp; 1799 1800 wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) & 1801 CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK; 1802 rq = &qp->rq; 1803 if (wr_id_idx > rq->hwq.max_elements) { 1804 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process RC "); 1805 dev_err(&cq->hwq.pdev->dev, 1806 "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x", 1807 wr_id_idx, rq->hwq.max_elements); 1808 return -EINVAL; 1809 } 1810 if (rq->flush_in_progress) 1811 goto flush_rq; 1812 1813 cqe->wr_id = rq->swq[wr_id_idx].wr_id; 1814 cqe++; 1815 (*budget)--; 1816 rq->hwq.cons++; 1817 *pcqe = cqe; 1818 1819 if (hwcqe->status != CQ_RES_RC_STATUS_OK) { 1820 rq->flush_in_progress = true; 1821 flush_rq: 1822 rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RC, pcqe, budget); 1823 if (!rc) 1824 rq->flush_in_progress = false; 1825 } 1826 return rc; 1827 } 1828 1829 static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq, 1830 struct cq_res_ud *hwcqe, 1831 struct bnxt_qplib_cqe **pcqe, 1832 int *budget) 1833 { 1834 struct bnxt_qplib_qp *qp; 1835 struct bnxt_qplib_q *rq; 1836 struct bnxt_qplib_cqe *cqe; 1837 u32 wr_id_idx; 1838 int rc = 0; 1839 1840 qp = (struct bnxt_qplib_qp *)((unsigned long) 1841 le64_to_cpu(hwcqe->qp_handle)); 1842 if (!qp) { 1843 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL"); 1844 return -EINVAL; 1845 } 1846 cqe = *pcqe; 1847 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; 1848 cqe->length = le32_to_cpu(hwcqe->length); 1849 cqe->invrkey = le32_to_cpu(hwcqe->imm_data); 1850 cqe->flags = le16_to_cpu(hwcqe->flags); 1851 cqe->status = hwcqe->status; 1852 cqe->qp_handle = (u64)(unsigned long)qp; 1853 memcpy(cqe->smac, hwcqe->src_mac, 6); 1854 wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) 1855 & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK; 1856 cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) | 1857 ((le32_to_cpu( 1858 hwcqe->src_qp_high_srq_or_rq_wr_id) & 1859 CQ_RES_UD_SRC_QP_HIGH_MASK) >> 8); 1860 1861 rq = &qp->rq; 1862 if (wr_id_idx > rq->hwq.max_elements) { 1863 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process UD "); 1864 dev_err(&cq->hwq.pdev->dev, 1865 "QPLIB: wr_id idx %#x exceeded RQ max %#x", 1866 wr_id_idx, rq->hwq.max_elements); 1867 return -EINVAL; 1868 } 1869 if (rq->flush_in_progress) 1870 goto flush_rq; 1871 1872 cqe->wr_id = rq->swq[wr_id_idx].wr_id; 1873 cqe++; 1874 (*budget)--; 1875 rq->hwq.cons++; 1876 *pcqe = cqe; 1877 1878 if (hwcqe->status != CQ_RES_RC_STATUS_OK) { 1879 rq->flush_in_progress = true; 1880 flush_rq: 1881 rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_UD, pcqe, budget); 1882 if (!rc) 1883 rq->flush_in_progress = false; 1884 } 1885 return rc; 1886 } 1887 1888 static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq, 1889 struct cq_res_raweth_qp1 *hwcqe, 1890 struct bnxt_qplib_cqe **pcqe, 1891 int *budget) 1892 { 1893 struct bnxt_qplib_qp *qp; 1894 struct bnxt_qplib_q *rq; 1895 struct bnxt_qplib_cqe *cqe; 1896 u32 wr_id_idx; 1897 int rc = 0; 1898 1899 qp = (struct bnxt_qplib_qp *)((unsigned long) 1900 le64_to_cpu(hwcqe->qp_handle)); 1901 if (!qp) { 1902 dev_err(&cq->hwq.pdev->dev, 1903 "QPLIB: process_cq Raw/QP1 qp is NULL"); 1904 return -EINVAL; 1905 } 1906 cqe = *pcqe; 1907 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; 1908 cqe->flags = le16_to_cpu(hwcqe->flags); 1909 cqe->qp_handle = (u64)(unsigned long)qp; 1910 1911 wr_id_idx = 1912 le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id) 1913 & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK; 1914 cqe->src_qp = qp->id; 1915 if (qp->id == 1 && !cqe->length) { 1916 /* Add workaround for the length misdetection */ 1917 cqe->length = 296; 1918 } else { 1919 cqe->length = le16_to_cpu(hwcqe->length); 1920 } 1921 cqe->pkey_index = qp->pkey_index; 1922 memcpy(cqe->smac, qp->smac, 6); 1923 1924 cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags); 1925 cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2); 1926 1927 rq = &qp->rq; 1928 if (wr_id_idx > rq->hwq.max_elements) { 1929 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id "); 1930 dev_err(&cq->hwq.pdev->dev, "QPLIB: ix 0x%x exceeded RQ max 0x%x", 1931 wr_id_idx, rq->hwq.max_elements); 1932 return -EINVAL; 1933 } 1934 if (rq->flush_in_progress) 1935 goto flush_rq; 1936 1937 cqe->wr_id = rq->swq[wr_id_idx].wr_id; 1938 cqe++; 1939 (*budget)--; 1940 rq->hwq.cons++; 1941 *pcqe = cqe; 1942 1943 if (hwcqe->status != CQ_RES_RC_STATUS_OK) { 1944 rq->flush_in_progress = true; 1945 flush_rq: 1946 rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RAWETH_QP1, pcqe, 1947 budget); 1948 if (!rc) 1949 rq->flush_in_progress = false; 1950 } 1951 return rc; 1952 } 1953 1954 static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, 1955 struct cq_terminal *hwcqe, 1956 struct bnxt_qplib_cqe **pcqe, 1957 int *budget) 1958 { 1959 struct bnxt_qplib_qp *qp; 1960 struct bnxt_qplib_q *sq, *rq; 1961 struct bnxt_qplib_cqe *cqe; 1962 u32 sw_cons = 0, cqe_cons; 1963 int rc = 0; 1964 u8 opcode = 0; 1965 1966 /* Check the Status */ 1967 if (hwcqe->status != CQ_TERMINAL_STATUS_OK) 1968 dev_warn(&cq->hwq.pdev->dev, 1969 "QPLIB: FP: CQ Process Terminal Error status = 0x%x", 1970 hwcqe->status); 1971 1972 qp = (struct bnxt_qplib_qp *)((unsigned long) 1973 le64_to_cpu(hwcqe->qp_handle)); 1974 if (!qp) { 1975 dev_err(&cq->hwq.pdev->dev, 1976 "QPLIB: FP: CQ Process terminal qp is NULL"); 1977 return -EINVAL; 1978 } 1979 /* Must block new posting of SQ and RQ */ 1980 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; 1981 1982 sq = &qp->sq; 1983 rq = &qp->rq; 1984 1985 cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx); 1986 if (cqe_cons == 0xFFFF) 1987 goto do_rq; 1988 1989 if (cqe_cons > sq->hwq.max_elements) { 1990 dev_err(&cq->hwq.pdev->dev, 1991 "QPLIB: FP: CQ Process terminal reported "); 1992 dev_err(&cq->hwq.pdev->dev, 1993 "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x", 1994 cqe_cons, sq->hwq.max_elements); 1995 goto do_rq; 1996 } 1997 /* If we were in the middle of flushing, continue */ 1998 if (sq->flush_in_progress) 1999 goto flush_sq; 2000 2001 /* Terminal CQE can also include aggregated successful CQEs prior. 2002 * So we must complete all CQEs from the current sq's cons to the 2003 * cq_cons with status OK 2004 */ 2005 cqe = *pcqe; 2006 while (*budget) { 2007 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); 2008 if (sw_cons == cqe_cons) 2009 break; 2010 if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) { 2011 memset(cqe, 0, sizeof(*cqe)); 2012 cqe->status = CQ_REQ_STATUS_OK; 2013 cqe->opcode = CQ_BASE_CQE_TYPE_REQ; 2014 cqe->qp_handle = (u64)(unsigned long)qp; 2015 cqe->src_qp = qp->id; 2016 cqe->wr_id = sq->swq[sw_cons].wr_id; 2017 cqe->type = sq->swq[sw_cons].type; 2018 cqe++; 2019 (*budget)--; 2020 } 2021 sq->hwq.cons++; 2022 } 2023 *pcqe = cqe; 2024 if (!(*budget) && sw_cons != cqe_cons) { 2025 /* Out of budget */ 2026 rc = -EAGAIN; 2027 goto sq_done; 2028 } 2029 sq->flush_in_progress = true; 2030 flush_sq: 2031 rc = __flush_sq(sq, qp, pcqe, budget); 2032 if (!rc) 2033 sq->flush_in_progress = false; 2034 sq_done: 2035 if (rc) 2036 return rc; 2037 do_rq: 2038 cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx); 2039 if (cqe_cons == 0xFFFF) { 2040 goto done; 2041 } else if (cqe_cons > rq->hwq.max_elements) { 2042 dev_err(&cq->hwq.pdev->dev, 2043 "QPLIB: FP: CQ Processed terminal "); 2044 dev_err(&cq->hwq.pdev->dev, 2045 "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x", 2046 cqe_cons, rq->hwq.max_elements); 2047 goto done; 2048 } 2049 /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR 2050 * from the current rq->cons to the rq->prod regardless what the 2051 * rq->cons the terminal CQE indicates 2052 */ 2053 rq->flush_in_progress = true; 2054 switch (qp->type) { 2055 case CMDQ_CREATE_QP1_TYPE_GSI: 2056 opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1; 2057 break; 2058 case CMDQ_CREATE_QP_TYPE_RC: 2059 opcode = CQ_BASE_CQE_TYPE_RES_RC; 2060 break; 2061 case CMDQ_CREATE_QP_TYPE_UD: 2062 opcode = CQ_BASE_CQE_TYPE_RES_UD; 2063 break; 2064 } 2065 2066 rc = __flush_rq(rq, qp, opcode, pcqe, budget); 2067 if (!rc) 2068 rq->flush_in_progress = false; 2069 done: 2070 return rc; 2071 } 2072 2073 static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq, 2074 struct cq_cutoff *hwcqe) 2075 { 2076 /* Check the Status */ 2077 if (hwcqe->status != CQ_CUTOFF_STATUS_OK) { 2078 dev_err(&cq->hwq.pdev->dev, 2079 "QPLIB: FP: CQ Process Cutoff Error status = 0x%x", 2080 hwcqe->status); 2081 return -EINVAL; 2082 } 2083 clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); 2084 wake_up_interruptible(&cq->waitq); 2085 2086 return 0; 2087 } 2088 2089 int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, 2090 int num_cqes, struct bnxt_qplib_qp **lib_qp) 2091 { 2092 struct cq_base *hw_cqe, **hw_cqe_ptr; 2093 unsigned long flags; 2094 u32 sw_cons, raw_cons; 2095 int budget, rc = 0; 2096 2097 spin_lock_irqsave(&cq->hwq.lock, flags); 2098 raw_cons = cq->hwq.cons; 2099 budget = num_cqes; 2100 2101 while (budget) { 2102 sw_cons = HWQ_CMP(raw_cons, &cq->hwq); 2103 hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr; 2104 hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)]; 2105 2106 /* Check for Valid bit */ 2107 if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements)) 2108 break; 2109 2110 /* From the device's respective CQE format to qplib_wc*/ 2111 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { 2112 case CQ_BASE_CQE_TYPE_REQ: 2113 rc = bnxt_qplib_cq_process_req(cq, 2114 (struct cq_req *)hw_cqe, 2115 &cqe, &budget, 2116 sw_cons, lib_qp); 2117 break; 2118 case CQ_BASE_CQE_TYPE_RES_RC: 2119 rc = bnxt_qplib_cq_process_res_rc(cq, 2120 (struct cq_res_rc *) 2121 hw_cqe, &cqe, 2122 &budget); 2123 break; 2124 case CQ_BASE_CQE_TYPE_RES_UD: 2125 rc = bnxt_qplib_cq_process_res_ud 2126 (cq, (struct cq_res_ud *)hw_cqe, &cqe, 2127 &budget); 2128 break; 2129 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: 2130 rc = bnxt_qplib_cq_process_res_raweth_qp1 2131 (cq, (struct cq_res_raweth_qp1 *) 2132 hw_cqe, &cqe, &budget); 2133 break; 2134 case CQ_BASE_CQE_TYPE_TERMINAL: 2135 rc = bnxt_qplib_cq_process_terminal 2136 (cq, (struct cq_terminal *)hw_cqe, 2137 &cqe, &budget); 2138 break; 2139 case CQ_BASE_CQE_TYPE_CUT_OFF: 2140 bnxt_qplib_cq_process_cutoff 2141 (cq, (struct cq_cutoff *)hw_cqe); 2142 /* Done processing this CQ */ 2143 goto exit; 2144 default: 2145 dev_err(&cq->hwq.pdev->dev, 2146 "QPLIB: process_cq unknown type 0x%lx", 2147 hw_cqe->cqe_type_toggle & 2148 CQ_BASE_CQE_TYPE_MASK); 2149 rc = -EINVAL; 2150 break; 2151 } 2152 if (rc < 0) { 2153 if (rc == -EAGAIN) 2154 break; 2155 /* Error while processing the CQE, just skip to the 2156 * next one 2157 */ 2158 dev_err(&cq->hwq.pdev->dev, 2159 "QPLIB: process_cqe error rc = 0x%x", rc); 2160 } 2161 raw_cons++; 2162 } 2163 if (cq->hwq.cons != raw_cons) { 2164 cq->hwq.cons = raw_cons; 2165 bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ); 2166 } 2167 exit: 2168 spin_unlock_irqrestore(&cq->hwq.lock, flags); 2169 return num_cqes - budget; 2170 } 2171 2172 void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type) 2173 { 2174 unsigned long flags; 2175 2176 spin_lock_irqsave(&cq->hwq.lock, flags); 2177 if (arm_type) 2178 bnxt_qplib_arm_cq(cq, arm_type); 2179 2180 spin_unlock_irqrestore(&cq->hwq.lock, flags); 2181 } 2182