1 /* 2 * Broadcom NetXtreme-E RoCE driver. 3 * 4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term 5 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the 22 * distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * Description: Fast Path Operators 37 */ 38 39 #include <linux/interrupt.h> 40 #include <linux/spinlock.h> 41 #include <linux/sched.h> 42 #include <linux/slab.h> 43 #include <linux/pci.h> 44 #include <linux/prefetch.h> 45 46 #include "roce_hsi.h" 47 48 #include "qplib_res.h" 49 #include "qplib_rcfw.h" 50 #include "qplib_sp.h" 51 #include "qplib_fp.h" 52 53 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq); 54 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp); 55 56 static void bnxt_qplib_cancel_phantom_processing(struct bnxt_qplib_qp *qp) 57 { 58 qp->sq.condition = false; 59 qp->sq.send_phantom = false; 60 qp->sq.single = false; 61 } 62 63 /* Flush list */ 64 static void __bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp) 65 { 66 struct bnxt_qplib_cq *scq, *rcq; 67 68 scq = qp->scq; 69 rcq = qp->rcq; 70 71 if (!qp->sq.flushed) { 72 dev_dbg(&scq->hwq.pdev->dev, 73 "QPLIB: FP: Adding to SQ Flush list = %p", 74 qp); 75 bnxt_qplib_cancel_phantom_processing(qp); 76 list_add_tail(&qp->sq_flush, &scq->sqf_head); 77 qp->sq.flushed = true; 78 } 79 if (!qp->srq) { 80 if (!qp->rq.flushed) { 81 dev_dbg(&rcq->hwq.pdev->dev, 82 "QPLIB: FP: Adding to RQ Flush list = %p", 83 qp); 84 list_add_tail(&qp->rq_flush, &rcq->rqf_head); 85 qp->rq.flushed = true; 86 } 87 } 88 } 89 90 void bnxt_qplib_acquire_cq_locks(struct bnxt_qplib_qp *qp, 91 unsigned long *flags) 92 __acquires(&qp->scq->hwq.lock) __acquires(&qp->rcq->hwq.lock) 93 { 94 spin_lock_irqsave(&qp->scq->hwq.lock, *flags); 95 if (qp->scq == qp->rcq) 96 __acquire(&qp->rcq->hwq.lock); 97 else 98 spin_lock(&qp->rcq->hwq.lock); 99 } 100 101 void bnxt_qplib_release_cq_locks(struct bnxt_qplib_qp *qp, 102 unsigned long *flags) 103 __releases(&qp->scq->hwq.lock) __releases(&qp->rcq->hwq.lock) 104 { 105 if (qp->scq == qp->rcq) 106 __release(&qp->rcq->hwq.lock); 107 else 108 spin_unlock(&qp->rcq->hwq.lock); 109 spin_unlock_irqrestore(&qp->scq->hwq.lock, *flags); 110 } 111 112 static struct bnxt_qplib_cq *bnxt_qplib_find_buddy_cq(struct bnxt_qplib_qp *qp, 113 struct bnxt_qplib_cq *cq) 114 { 115 struct bnxt_qplib_cq *buddy_cq = NULL; 116 117 if (qp->scq == qp->rcq) 118 buddy_cq = NULL; 119 else if (qp->scq == cq) 120 buddy_cq = qp->rcq; 121 else 122 buddy_cq = qp->scq; 123 return buddy_cq; 124 } 125 126 static void bnxt_qplib_lock_buddy_cq(struct bnxt_qplib_qp *qp, 127 struct bnxt_qplib_cq *cq) 128 __acquires(&buddy_cq->hwq.lock) 129 { 130 struct bnxt_qplib_cq *buddy_cq = NULL; 131 132 buddy_cq = bnxt_qplib_find_buddy_cq(qp, cq); 133 if (!buddy_cq) 134 __acquire(&cq->hwq.lock); 135 else 136 spin_lock(&buddy_cq->hwq.lock); 137 } 138 139 static void bnxt_qplib_unlock_buddy_cq(struct bnxt_qplib_qp *qp, 140 struct bnxt_qplib_cq *cq) 141 __releases(&buddy_cq->hwq.lock) 142 { 143 struct bnxt_qplib_cq *buddy_cq = NULL; 144 145 buddy_cq = bnxt_qplib_find_buddy_cq(qp, cq); 146 if (!buddy_cq) 147 __release(&cq->hwq.lock); 148 else 149 spin_unlock(&buddy_cq->hwq.lock); 150 } 151 152 void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp) 153 { 154 unsigned long flags; 155 156 bnxt_qplib_acquire_cq_locks(qp, &flags); 157 __bnxt_qplib_add_flush_qp(qp); 158 bnxt_qplib_release_cq_locks(qp, &flags); 159 } 160 161 static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp) 162 { 163 if (qp->sq.flushed) { 164 qp->sq.flushed = false; 165 list_del(&qp->sq_flush); 166 } 167 if (!qp->srq) { 168 if (qp->rq.flushed) { 169 qp->rq.flushed = false; 170 list_del(&qp->rq_flush); 171 } 172 } 173 } 174 175 void bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp) 176 { 177 unsigned long flags; 178 179 bnxt_qplib_acquire_cq_locks(qp, &flags); 180 __clean_cq(qp->scq, (u64)(unsigned long)qp); 181 qp->sq.hwq.prod = 0; 182 qp->sq.hwq.cons = 0; 183 __clean_cq(qp->rcq, (u64)(unsigned long)qp); 184 qp->rq.hwq.prod = 0; 185 qp->rq.hwq.cons = 0; 186 187 __bnxt_qplib_del_flush_qp(qp); 188 bnxt_qplib_release_cq_locks(qp, &flags); 189 } 190 191 static void bnxt_qpn_cqn_sched_task(struct work_struct *work) 192 { 193 struct bnxt_qplib_nq_work *nq_work = 194 container_of(work, struct bnxt_qplib_nq_work, work); 195 196 struct bnxt_qplib_cq *cq = nq_work->cq; 197 struct bnxt_qplib_nq *nq = nq_work->nq; 198 199 if (cq && nq) { 200 spin_lock_bh(&cq->compl_lock); 201 if (atomic_read(&cq->arm_state) && nq->cqn_handler) { 202 dev_dbg(&nq->pdev->dev, 203 "%s:Trigger cq = %p event nq = %p\n", 204 __func__, cq, nq); 205 nq->cqn_handler(nq, cq); 206 } 207 spin_unlock_bh(&cq->compl_lock); 208 } 209 kfree(nq_work); 210 } 211 212 static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res, 213 struct bnxt_qplib_qp *qp) 214 { 215 struct bnxt_qplib_q *rq = &qp->rq; 216 struct bnxt_qplib_q *sq = &qp->sq; 217 218 if (qp->rq_hdr_buf) 219 dma_free_coherent(&res->pdev->dev, 220 rq->hwq.max_elements * qp->rq_hdr_buf_size, 221 qp->rq_hdr_buf, qp->rq_hdr_buf_map); 222 if (qp->sq_hdr_buf) 223 dma_free_coherent(&res->pdev->dev, 224 sq->hwq.max_elements * qp->sq_hdr_buf_size, 225 qp->sq_hdr_buf, qp->sq_hdr_buf_map); 226 qp->rq_hdr_buf = NULL; 227 qp->sq_hdr_buf = NULL; 228 qp->rq_hdr_buf_map = 0; 229 qp->sq_hdr_buf_map = 0; 230 qp->sq_hdr_buf_size = 0; 231 qp->rq_hdr_buf_size = 0; 232 } 233 234 static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res, 235 struct bnxt_qplib_qp *qp) 236 { 237 struct bnxt_qplib_q *rq = &qp->rq; 238 struct bnxt_qplib_q *sq = &qp->rq; 239 int rc = 0; 240 241 if (qp->sq_hdr_buf_size && sq->hwq.max_elements) { 242 qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, 243 sq->hwq.max_elements * 244 qp->sq_hdr_buf_size, 245 &qp->sq_hdr_buf_map, GFP_KERNEL); 246 if (!qp->sq_hdr_buf) { 247 rc = -ENOMEM; 248 dev_err(&res->pdev->dev, 249 "QPLIB: Failed to create sq_hdr_buf"); 250 goto fail; 251 } 252 } 253 254 if (qp->rq_hdr_buf_size && rq->hwq.max_elements) { 255 qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, 256 rq->hwq.max_elements * 257 qp->rq_hdr_buf_size, 258 &qp->rq_hdr_buf_map, 259 GFP_KERNEL); 260 if (!qp->rq_hdr_buf) { 261 rc = -ENOMEM; 262 dev_err(&res->pdev->dev, 263 "QPLIB: Failed to create rq_hdr_buf"); 264 goto fail; 265 } 266 } 267 return 0; 268 269 fail: 270 bnxt_qplib_free_qp_hdr_buf(res, qp); 271 return rc; 272 } 273 274 static void bnxt_qplib_service_nq(unsigned long data) 275 { 276 struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data; 277 struct bnxt_qplib_hwq *hwq = &nq->hwq; 278 struct nq_base *nqe, **nq_ptr; 279 struct bnxt_qplib_cq *cq; 280 int num_cqne_processed = 0; 281 u32 sw_cons, raw_cons; 282 u16 type; 283 int budget = nq->budget; 284 u64 q_handle; 285 286 /* Service the NQ until empty */ 287 raw_cons = hwq->cons; 288 while (budget--) { 289 sw_cons = HWQ_CMP(raw_cons, hwq); 290 nq_ptr = (struct nq_base **)hwq->pbl_ptr; 291 nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]; 292 if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements)) 293 break; 294 295 /* 296 * The valid test of the entry must be done first before 297 * reading any further. 298 */ 299 dma_rmb(); 300 301 type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK; 302 switch (type) { 303 case NQ_BASE_TYPE_CQ_NOTIFICATION: 304 { 305 struct nq_cn *nqcne = (struct nq_cn *)nqe; 306 307 q_handle = le32_to_cpu(nqcne->cq_handle_low); 308 q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) 309 << 32; 310 cq = (struct bnxt_qplib_cq *)(unsigned long)q_handle; 311 bnxt_qplib_arm_cq_enable(cq); 312 spin_lock_bh(&cq->compl_lock); 313 atomic_set(&cq->arm_state, 0); 314 if (!nq->cqn_handler(nq, (cq))) 315 num_cqne_processed++; 316 else 317 dev_warn(&nq->pdev->dev, 318 "QPLIB: cqn - type 0x%x not handled", 319 type); 320 spin_unlock_bh(&cq->compl_lock); 321 break; 322 } 323 case NQ_BASE_TYPE_DBQ_EVENT: 324 break; 325 default: 326 dev_warn(&nq->pdev->dev, 327 "QPLIB: nqe with type = 0x%x not handled", 328 type); 329 break; 330 } 331 raw_cons++; 332 } 333 if (hwq->cons != raw_cons) { 334 hwq->cons = raw_cons; 335 NQ_DB_REARM(nq->bar_reg_iomem, hwq->cons, hwq->max_elements); 336 } 337 } 338 339 static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance) 340 { 341 struct bnxt_qplib_nq *nq = dev_instance; 342 struct bnxt_qplib_hwq *hwq = &nq->hwq; 343 struct nq_base **nq_ptr; 344 u32 sw_cons; 345 346 /* Prefetch the NQ element */ 347 sw_cons = HWQ_CMP(hwq->cons, hwq); 348 nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr; 349 prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]); 350 351 /* Fan out to CPU affinitized kthreads? */ 352 tasklet_schedule(&nq->worker); 353 354 return IRQ_HANDLED; 355 } 356 357 void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq) 358 { 359 if (nq->cqn_wq) { 360 destroy_workqueue(nq->cqn_wq); 361 nq->cqn_wq = NULL; 362 } 363 /* Make sure the HW is stopped! */ 364 synchronize_irq(nq->vector); 365 tasklet_disable(&nq->worker); 366 tasklet_kill(&nq->worker); 367 368 if (nq->requested) { 369 irq_set_affinity_hint(nq->vector, NULL); 370 free_irq(nq->vector, nq); 371 nq->requested = false; 372 } 373 if (nq->bar_reg_iomem) 374 iounmap(nq->bar_reg_iomem); 375 nq->bar_reg_iomem = NULL; 376 377 nq->cqn_handler = NULL; 378 nq->srqn_handler = NULL; 379 nq->vector = 0; 380 } 381 382 int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq, 383 int nq_idx, int msix_vector, int bar_reg_offset, 384 int (*cqn_handler)(struct bnxt_qplib_nq *nq, 385 struct bnxt_qplib_cq *), 386 int (*srqn_handler)(struct bnxt_qplib_nq *nq, 387 void *, u8 event)) 388 { 389 resource_size_t nq_base; 390 int rc = -1; 391 392 nq->pdev = pdev; 393 nq->vector = msix_vector; 394 395 nq->cqn_handler = cqn_handler; 396 397 nq->srqn_handler = srqn_handler; 398 399 tasklet_init(&nq->worker, bnxt_qplib_service_nq, (unsigned long)nq); 400 401 /* Have a task to schedule CQ notifiers in post send case */ 402 nq->cqn_wq = create_singlethread_workqueue("bnxt_qplib_nq"); 403 if (!nq->cqn_wq) 404 goto fail; 405 406 nq->requested = false; 407 memset(nq->name, 0, 32); 408 sprintf(nq->name, "bnxt_qplib_nq-%d", nq_idx); 409 rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, nq->name, nq); 410 if (rc) { 411 dev_err(&nq->pdev->dev, 412 "Failed to request IRQ for NQ: %#x", rc); 413 bnxt_qplib_disable_nq(nq); 414 goto fail; 415 } 416 417 cpumask_clear(&nq->mask); 418 cpumask_set_cpu(nq_idx, &nq->mask); 419 rc = irq_set_affinity_hint(nq->vector, &nq->mask); 420 if (rc) { 421 dev_warn(&nq->pdev->dev, 422 "QPLIB: set affinity failed; vector: %d nq_idx: %d\n", 423 nq->vector, nq_idx); 424 } 425 426 nq->requested = true; 427 nq->bar_reg = NQ_CONS_PCI_BAR_REGION; 428 nq->bar_reg_off = bar_reg_offset; 429 nq_base = pci_resource_start(pdev, nq->bar_reg); 430 if (!nq_base) { 431 rc = -ENOMEM; 432 goto fail; 433 } 434 nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 4); 435 if (!nq->bar_reg_iomem) { 436 rc = -ENOMEM; 437 goto fail; 438 } 439 NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements); 440 441 return 0; 442 fail: 443 bnxt_qplib_disable_nq(nq); 444 return rc; 445 } 446 447 void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq) 448 { 449 if (nq->hwq.max_elements) { 450 bnxt_qplib_free_hwq(nq->pdev, &nq->hwq); 451 nq->hwq.max_elements = 0; 452 } 453 } 454 455 int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq) 456 { 457 nq->pdev = pdev; 458 if (!nq->hwq.max_elements || 459 nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT) 460 nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT; 461 462 if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0, 463 &nq->hwq.max_elements, 464 BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0, 465 PAGE_SIZE, HWQ_TYPE_L2_CMPL)) 466 return -ENOMEM; 467 468 nq->budget = 8; 469 return 0; 470 } 471 472 /* QP */ 473 int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) 474 { 475 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 476 struct cmdq_create_qp1 req; 477 struct creq_create_qp1_resp resp; 478 struct bnxt_qplib_pbl *pbl; 479 struct bnxt_qplib_q *sq = &qp->sq; 480 struct bnxt_qplib_q *rq = &qp->rq; 481 int rc; 482 u16 cmd_flags = 0; 483 u32 qp_flags = 0; 484 485 RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags); 486 487 /* General */ 488 req.type = qp->type; 489 req.dpi = cpu_to_le32(qp->dpi->dpi); 490 req.qp_handle = cpu_to_le64(qp->qp_handle); 491 492 /* SQ */ 493 sq->hwq.max_elements = sq->max_wqe; 494 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL, 0, 495 &sq->hwq.max_elements, 496 BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0, 497 PAGE_SIZE, HWQ_TYPE_QUEUE); 498 if (rc) 499 goto exit; 500 501 sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL); 502 if (!sq->swq) { 503 rc = -ENOMEM; 504 goto fail_sq; 505 } 506 pbl = &sq->hwq.pbl[PBL_LVL_0]; 507 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); 508 req.sq_pg_size_sq_lvl = 509 ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK) 510 << CMDQ_CREATE_QP1_SQ_LVL_SFT) | 511 (pbl->pg_size == ROCE_PG_SIZE_4K ? 512 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K : 513 pbl->pg_size == ROCE_PG_SIZE_8K ? 514 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K : 515 pbl->pg_size == ROCE_PG_SIZE_64K ? 516 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K : 517 pbl->pg_size == ROCE_PG_SIZE_2M ? 518 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M : 519 pbl->pg_size == ROCE_PG_SIZE_8M ? 520 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M : 521 pbl->pg_size == ROCE_PG_SIZE_1G ? 522 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G : 523 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K); 524 525 if (qp->scq) 526 req.scq_cid = cpu_to_le32(qp->scq->id); 527 528 qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE; 529 530 /* RQ */ 531 if (rq->max_wqe) { 532 rq->hwq.max_elements = qp->rq.max_wqe; 533 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL, 0, 534 &rq->hwq.max_elements, 535 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0, 536 PAGE_SIZE, HWQ_TYPE_QUEUE); 537 if (rc) 538 goto fail_sq; 539 540 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq), 541 GFP_KERNEL); 542 if (!rq->swq) { 543 rc = -ENOMEM; 544 goto fail_rq; 545 } 546 pbl = &rq->hwq.pbl[PBL_LVL_0]; 547 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); 548 req.rq_pg_size_rq_lvl = 549 ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) << 550 CMDQ_CREATE_QP1_RQ_LVL_SFT) | 551 (pbl->pg_size == ROCE_PG_SIZE_4K ? 552 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K : 553 pbl->pg_size == ROCE_PG_SIZE_8K ? 554 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K : 555 pbl->pg_size == ROCE_PG_SIZE_64K ? 556 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K : 557 pbl->pg_size == ROCE_PG_SIZE_2M ? 558 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M : 559 pbl->pg_size == ROCE_PG_SIZE_8M ? 560 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M : 561 pbl->pg_size == ROCE_PG_SIZE_1G ? 562 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G : 563 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K); 564 if (qp->rcq) 565 req.rcq_cid = cpu_to_le32(qp->rcq->id); 566 } 567 568 /* Header buffer - allow hdr_buf pass in */ 569 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); 570 if (rc) { 571 rc = -ENOMEM; 572 goto fail; 573 } 574 req.qp_flags = cpu_to_le32(qp_flags); 575 req.sq_size = cpu_to_le32(sq->hwq.max_elements); 576 req.rq_size = cpu_to_le32(rq->hwq.max_elements); 577 578 req.sq_fwo_sq_sge = 579 cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) << 580 CMDQ_CREATE_QP1_SQ_SGE_SFT); 581 req.rq_fwo_rq_sge = 582 cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) << 583 CMDQ_CREATE_QP1_RQ_SGE_SFT); 584 585 req.pd_id = cpu_to_le32(qp->pd->id); 586 587 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 588 (void *)&resp, NULL, 0); 589 if (rc) 590 goto fail; 591 592 qp->id = le32_to_cpu(resp.xid); 593 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; 594 rcfw->qp_tbl[qp->id].qp_id = qp->id; 595 rcfw->qp_tbl[qp->id].qp_handle = (void *)qp; 596 597 return 0; 598 599 fail: 600 bnxt_qplib_free_qp_hdr_buf(res, qp); 601 fail_rq: 602 bnxt_qplib_free_hwq(res->pdev, &rq->hwq); 603 kfree(rq->swq); 604 fail_sq: 605 bnxt_qplib_free_hwq(res->pdev, &sq->hwq); 606 kfree(sq->swq); 607 exit: 608 return rc; 609 } 610 611 int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) 612 { 613 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 614 struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr; 615 struct cmdq_create_qp req; 616 struct creq_create_qp_resp resp; 617 struct bnxt_qplib_pbl *pbl; 618 struct sq_psn_search **psn_search_ptr; 619 unsigned long int psn_search, poff = 0; 620 struct bnxt_qplib_q *sq = &qp->sq; 621 struct bnxt_qplib_q *rq = &qp->rq; 622 struct bnxt_qplib_hwq *xrrq; 623 int i, rc, req_size, psn_sz; 624 u16 cmd_flags = 0, max_ssge; 625 u32 sw_prod, qp_flags = 0; 626 627 RCFW_CMD_PREP(req, CREATE_QP, cmd_flags); 628 629 /* General */ 630 req.type = qp->type; 631 req.dpi = cpu_to_le32(qp->dpi->dpi); 632 req.qp_handle = cpu_to_le64(qp->qp_handle); 633 634 /* SQ */ 635 psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ? 636 sizeof(struct sq_psn_search) : 0; 637 sq->hwq.max_elements = sq->max_wqe; 638 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist, 639 sq->nmap, &sq->hwq.max_elements, 640 BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 641 psn_sz, 642 PAGE_SIZE, HWQ_TYPE_QUEUE); 643 if (rc) 644 goto exit; 645 646 sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL); 647 if (!sq->swq) { 648 rc = -ENOMEM; 649 goto fail_sq; 650 } 651 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; 652 if (psn_sz) { 653 psn_search_ptr = (struct sq_psn_search **) 654 &hw_sq_send_ptr[get_sqe_pg 655 (sq->hwq.max_elements)]; 656 psn_search = (unsigned long int) 657 &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)] 658 [get_sqe_idx(sq->hwq.max_elements)]; 659 if (psn_search & ~PAGE_MASK) { 660 /* If the psn_search does not start on a page boundary, 661 * then calculate the offset 662 */ 663 poff = (psn_search & ~PAGE_MASK) / 664 BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE; 665 } 666 for (i = 0; i < sq->hwq.max_elements; i++) 667 sq->swq[i].psn_search = 668 &psn_search_ptr[get_psne_pg(i + poff)] 669 [get_psne_idx(i + poff)]; 670 } 671 pbl = &sq->hwq.pbl[PBL_LVL_0]; 672 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); 673 req.sq_pg_size_sq_lvl = 674 ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK) 675 << CMDQ_CREATE_QP_SQ_LVL_SFT) | 676 (pbl->pg_size == ROCE_PG_SIZE_4K ? 677 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K : 678 pbl->pg_size == ROCE_PG_SIZE_8K ? 679 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K : 680 pbl->pg_size == ROCE_PG_SIZE_64K ? 681 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K : 682 pbl->pg_size == ROCE_PG_SIZE_2M ? 683 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M : 684 pbl->pg_size == ROCE_PG_SIZE_8M ? 685 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M : 686 pbl->pg_size == ROCE_PG_SIZE_1G ? 687 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G : 688 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K); 689 690 /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */ 691 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; 692 for (sw_prod = 0; sw_prod < sq->hwq.max_elements; sw_prod++) { 693 hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)] 694 [get_sqe_idx(sw_prod)]; 695 hw_sq_send_hdr->wqe_type = SQ_BASE_WQE_TYPE_LOCAL_INVALID; 696 } 697 698 if (qp->scq) 699 req.scq_cid = cpu_to_le32(qp->scq->id); 700 701 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE; 702 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED; 703 if (qp->sig_type) 704 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION; 705 706 /* RQ */ 707 if (rq->max_wqe) { 708 rq->hwq.max_elements = rq->max_wqe; 709 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, rq->sglist, 710 rq->nmap, &rq->hwq.max_elements, 711 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0, 712 PAGE_SIZE, HWQ_TYPE_QUEUE); 713 if (rc) 714 goto fail_sq; 715 716 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq), 717 GFP_KERNEL); 718 if (!rq->swq) { 719 rc = -ENOMEM; 720 goto fail_rq; 721 } 722 pbl = &rq->hwq.pbl[PBL_LVL_0]; 723 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); 724 req.rq_pg_size_rq_lvl = 725 ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) << 726 CMDQ_CREATE_QP_RQ_LVL_SFT) | 727 (pbl->pg_size == ROCE_PG_SIZE_4K ? 728 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K : 729 pbl->pg_size == ROCE_PG_SIZE_8K ? 730 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K : 731 pbl->pg_size == ROCE_PG_SIZE_64K ? 732 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K : 733 pbl->pg_size == ROCE_PG_SIZE_2M ? 734 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M : 735 pbl->pg_size == ROCE_PG_SIZE_8M ? 736 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M : 737 pbl->pg_size == ROCE_PG_SIZE_1G ? 738 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G : 739 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K); 740 } 741 742 if (qp->rcq) 743 req.rcq_cid = cpu_to_le32(qp->rcq->id); 744 req.qp_flags = cpu_to_le32(qp_flags); 745 req.sq_size = cpu_to_le32(sq->hwq.max_elements); 746 req.rq_size = cpu_to_le32(rq->hwq.max_elements); 747 qp->sq_hdr_buf = NULL; 748 qp->rq_hdr_buf = NULL; 749 750 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); 751 if (rc) 752 goto fail_rq; 753 754 /* CTRL-22434: Irrespective of the requested SGE count on the SQ 755 * always create the QP with max send sges possible if the requested 756 * inline size is greater than 0. 757 */ 758 max_ssge = qp->max_inline_data ? 6 : sq->max_sge; 759 req.sq_fwo_sq_sge = cpu_to_le16( 760 ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK) 761 << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0); 762 req.rq_fwo_rq_sge = cpu_to_le16( 763 ((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK) 764 << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0); 765 /* ORRQ and IRRQ */ 766 if (psn_sz) { 767 xrrq = &qp->orrq; 768 xrrq->max_elements = 769 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); 770 req_size = xrrq->max_elements * 771 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1; 772 req_size &= ~(PAGE_SIZE - 1); 773 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0, 774 &xrrq->max_elements, 775 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE, 776 0, req_size, HWQ_TYPE_CTX); 777 if (rc) 778 goto fail_buf_free; 779 pbl = &xrrq->pbl[PBL_LVL_0]; 780 req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]); 781 782 xrrq = &qp->irrq; 783 xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS( 784 qp->max_dest_rd_atomic); 785 req_size = xrrq->max_elements * 786 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1; 787 req_size &= ~(PAGE_SIZE - 1); 788 789 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0, 790 &xrrq->max_elements, 791 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE, 792 0, req_size, HWQ_TYPE_CTX); 793 if (rc) 794 goto fail_orrq; 795 796 pbl = &xrrq->pbl[PBL_LVL_0]; 797 req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]); 798 } 799 req.pd_id = cpu_to_le32(qp->pd->id); 800 801 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 802 (void *)&resp, NULL, 0); 803 if (rc) 804 goto fail; 805 806 qp->id = le32_to_cpu(resp.xid); 807 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; 808 INIT_LIST_HEAD(&qp->sq_flush); 809 INIT_LIST_HEAD(&qp->rq_flush); 810 rcfw->qp_tbl[qp->id].qp_id = qp->id; 811 rcfw->qp_tbl[qp->id].qp_handle = (void *)qp; 812 813 return 0; 814 815 fail: 816 if (qp->irrq.max_elements) 817 bnxt_qplib_free_hwq(res->pdev, &qp->irrq); 818 fail_orrq: 819 if (qp->orrq.max_elements) 820 bnxt_qplib_free_hwq(res->pdev, &qp->orrq); 821 fail_buf_free: 822 bnxt_qplib_free_qp_hdr_buf(res, qp); 823 fail_rq: 824 bnxt_qplib_free_hwq(res->pdev, &rq->hwq); 825 kfree(rq->swq); 826 fail_sq: 827 bnxt_qplib_free_hwq(res->pdev, &sq->hwq); 828 kfree(sq->swq); 829 exit: 830 return rc; 831 } 832 833 static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp) 834 { 835 switch (qp->state) { 836 case CMDQ_MODIFY_QP_NEW_STATE_RTR: 837 /* INIT->RTR, configure the path_mtu to the default 838 * 2048 if not being requested 839 */ 840 if (!(qp->modify_flags & 841 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) { 842 qp->modify_flags |= 843 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; 844 qp->path_mtu = 845 CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; 846 } 847 qp->modify_flags &= 848 ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; 849 /* Bono FW require the max_dest_rd_atomic to be >= 1 */ 850 if (qp->max_dest_rd_atomic < 1) 851 qp->max_dest_rd_atomic = 1; 852 qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC; 853 /* Bono FW 20.6.5 requires SGID_INDEX configuration */ 854 if (!(qp->modify_flags & 855 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) { 856 qp->modify_flags |= 857 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX; 858 qp->ah.sgid_index = 0; 859 } 860 break; 861 default: 862 break; 863 } 864 } 865 866 static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp *qp) 867 { 868 switch (qp->state) { 869 case CMDQ_MODIFY_QP_NEW_STATE_RTS: 870 /* Bono FW requires the max_rd_atomic to be >= 1 */ 871 if (qp->max_rd_atomic < 1) 872 qp->max_rd_atomic = 1; 873 /* Bono FW does not allow PKEY_INDEX, 874 * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT, 875 * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN, 876 * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID 877 * modification 878 */ 879 qp->modify_flags &= 880 ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY | 881 CMDQ_MODIFY_QP_MODIFY_MASK_DGID | 882 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | 883 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | 884 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | 885 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | 886 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | 887 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU | 888 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN | 889 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER | 890 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC | 891 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID); 892 break; 893 default: 894 break; 895 } 896 } 897 898 static void __filter_modify_flags(struct bnxt_qplib_qp *qp) 899 { 900 switch (qp->cur_qp_state) { 901 case CMDQ_MODIFY_QP_NEW_STATE_RESET: 902 break; 903 case CMDQ_MODIFY_QP_NEW_STATE_INIT: 904 __modify_flags_from_init_state(qp); 905 break; 906 case CMDQ_MODIFY_QP_NEW_STATE_RTR: 907 __modify_flags_from_rtr_state(qp); 908 break; 909 case CMDQ_MODIFY_QP_NEW_STATE_RTS: 910 break; 911 case CMDQ_MODIFY_QP_NEW_STATE_SQD: 912 break; 913 case CMDQ_MODIFY_QP_NEW_STATE_SQE: 914 break; 915 case CMDQ_MODIFY_QP_NEW_STATE_ERR: 916 break; 917 default: 918 break; 919 } 920 } 921 922 int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) 923 { 924 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 925 struct cmdq_modify_qp req; 926 struct creq_modify_qp_resp resp; 927 u16 cmd_flags = 0, pkey; 928 u32 temp32[4]; 929 u32 bmask; 930 int rc; 931 932 RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags); 933 934 /* Filter out the qp_attr_mask based on the state->new transition */ 935 __filter_modify_flags(qp); 936 bmask = qp->modify_flags; 937 req.modify_mask = cpu_to_le32(qp->modify_flags); 938 req.qp_cid = cpu_to_le32(qp->id); 939 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) { 940 req.network_type_en_sqd_async_notify_new_state = 941 (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) | 942 (qp->en_sqd_async_notify ? 943 CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0); 944 } 945 req.network_type_en_sqd_async_notify_new_state |= qp->nw_type; 946 947 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS) 948 req.access = qp->access; 949 950 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) { 951 if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl, 952 qp->pkey_index, &pkey)) 953 req.pkey = cpu_to_le16(pkey); 954 } 955 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY) 956 req.qkey = cpu_to_le32(qp->qkey); 957 958 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) { 959 memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid)); 960 req.dgid[0] = cpu_to_le32(temp32[0]); 961 req.dgid[1] = cpu_to_le32(temp32[1]); 962 req.dgid[2] = cpu_to_le32(temp32[2]); 963 req.dgid[3] = cpu_to_le32(temp32[3]); 964 } 965 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL) 966 req.flow_label = cpu_to_le32(qp->ah.flow_label); 967 968 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX) 969 req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id 970 [qp->ah.sgid_index]); 971 972 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT) 973 req.hop_limit = qp->ah.hop_limit; 974 975 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS) 976 req.traffic_class = qp->ah.traffic_class; 977 978 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC) 979 memcpy(req.dest_mac, qp->ah.dmac, 6); 980 981 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU) 982 req.path_mtu = qp->path_mtu; 983 984 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT) 985 req.timeout = qp->timeout; 986 987 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT) 988 req.retry_cnt = qp->retry_cnt; 989 990 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY) 991 req.rnr_retry = qp->rnr_retry; 992 993 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER) 994 req.min_rnr_timer = qp->min_rnr_timer; 995 996 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN) 997 req.rq_psn = cpu_to_le32(qp->rq.psn); 998 999 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN) 1000 req.sq_psn = cpu_to_le32(qp->sq.psn); 1001 1002 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC) 1003 req.max_rd_atomic = 1004 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); 1005 1006 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC) 1007 req.max_dest_rd_atomic = 1008 IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic); 1009 1010 req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements); 1011 req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements); 1012 req.sq_sge = cpu_to_le16(qp->sq.max_sge); 1013 req.rq_sge = cpu_to_le16(qp->rq.max_sge); 1014 req.max_inline_data = cpu_to_le32(qp->max_inline_data); 1015 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID) 1016 req.dest_qp_id = cpu_to_le32(qp->dest_qpn); 1017 1018 req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id); 1019 1020 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 1021 (void *)&resp, NULL, 0); 1022 if (rc) 1023 return rc; 1024 qp->cur_qp_state = qp->state; 1025 return 0; 1026 } 1027 1028 int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) 1029 { 1030 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 1031 struct cmdq_query_qp req; 1032 struct creq_query_qp_resp resp; 1033 struct bnxt_qplib_rcfw_sbuf *sbuf; 1034 struct creq_query_qp_resp_sb *sb; 1035 u16 cmd_flags = 0; 1036 u32 temp32[4]; 1037 int i, rc = 0; 1038 1039 RCFW_CMD_PREP(req, QUERY_QP, cmd_flags); 1040 1041 sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb)); 1042 if (!sbuf) 1043 return -ENOMEM; 1044 sb = sbuf->sb; 1045 1046 req.qp_cid = cpu_to_le32(qp->id); 1047 req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS; 1048 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, 1049 (void *)sbuf, 0); 1050 if (rc) 1051 goto bail; 1052 /* Extract the context from the side buffer */ 1053 qp->state = sb->en_sqd_async_notify_state & 1054 CREQ_QUERY_QP_RESP_SB_STATE_MASK; 1055 qp->en_sqd_async_notify = sb->en_sqd_async_notify_state & 1056 CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ? 1057 true : false; 1058 qp->access = sb->access; 1059 qp->pkey_index = le16_to_cpu(sb->pkey); 1060 qp->qkey = le32_to_cpu(sb->qkey); 1061 1062 temp32[0] = le32_to_cpu(sb->dgid[0]); 1063 temp32[1] = le32_to_cpu(sb->dgid[1]); 1064 temp32[2] = le32_to_cpu(sb->dgid[2]); 1065 temp32[3] = le32_to_cpu(sb->dgid[3]); 1066 memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data)); 1067 1068 qp->ah.flow_label = le32_to_cpu(sb->flow_label); 1069 1070 qp->ah.sgid_index = 0; 1071 for (i = 0; i < res->sgid_tbl.max; i++) { 1072 if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) { 1073 qp->ah.sgid_index = i; 1074 break; 1075 } 1076 } 1077 if (i == res->sgid_tbl.max) 1078 dev_warn(&res->pdev->dev, "QPLIB: SGID not found??"); 1079 1080 qp->ah.hop_limit = sb->hop_limit; 1081 qp->ah.traffic_class = sb->traffic_class; 1082 memcpy(qp->ah.dmac, sb->dest_mac, 6); 1083 qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) & 1084 CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK) >> 1085 CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT; 1086 qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) & 1087 CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) >> 1088 CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT; 1089 qp->timeout = sb->timeout; 1090 qp->retry_cnt = sb->retry_cnt; 1091 qp->rnr_retry = sb->rnr_retry; 1092 qp->min_rnr_timer = sb->min_rnr_timer; 1093 qp->rq.psn = le32_to_cpu(sb->rq_psn); 1094 qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic); 1095 qp->sq.psn = le32_to_cpu(sb->sq_psn); 1096 qp->max_dest_rd_atomic = 1097 IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic); 1098 qp->sq.max_wqe = qp->sq.hwq.max_elements; 1099 qp->rq.max_wqe = qp->rq.hwq.max_elements; 1100 qp->sq.max_sge = le16_to_cpu(sb->sq_sge); 1101 qp->rq.max_sge = le16_to_cpu(sb->rq_sge); 1102 qp->max_inline_data = le32_to_cpu(sb->max_inline_data); 1103 qp->dest_qpn = le32_to_cpu(sb->dest_qp_id); 1104 memcpy(qp->smac, sb->src_mac, 6); 1105 qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id); 1106 bail: 1107 bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf); 1108 return rc; 1109 } 1110 1111 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp) 1112 { 1113 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; 1114 struct cq_base *hw_cqe, **hw_cqe_ptr; 1115 int i; 1116 1117 for (i = 0; i < cq_hwq->max_elements; i++) { 1118 hw_cqe_ptr = (struct cq_base **)cq_hwq->pbl_ptr; 1119 hw_cqe = &hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)]; 1120 if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements)) 1121 continue; 1122 /* 1123 * The valid test of the entry must be done first before 1124 * reading any further. 1125 */ 1126 dma_rmb(); 1127 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { 1128 case CQ_BASE_CQE_TYPE_REQ: 1129 case CQ_BASE_CQE_TYPE_TERMINAL: 1130 { 1131 struct cq_req *cqe = (struct cq_req *)hw_cqe; 1132 1133 if (qp == le64_to_cpu(cqe->qp_handle)) 1134 cqe->qp_handle = 0; 1135 break; 1136 } 1137 case CQ_BASE_CQE_TYPE_RES_RC: 1138 case CQ_BASE_CQE_TYPE_RES_UD: 1139 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: 1140 { 1141 struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe; 1142 1143 if (qp == le64_to_cpu(cqe->qp_handle)) 1144 cqe->qp_handle = 0; 1145 break; 1146 } 1147 default: 1148 break; 1149 } 1150 } 1151 } 1152 1153 int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, 1154 struct bnxt_qplib_qp *qp) 1155 { 1156 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 1157 struct cmdq_destroy_qp req; 1158 struct creq_destroy_qp_resp resp; 1159 unsigned long flags; 1160 u16 cmd_flags = 0; 1161 int rc; 1162 1163 rcfw->qp_tbl[qp->id].qp_id = BNXT_QPLIB_QP_ID_INVALID; 1164 rcfw->qp_tbl[qp->id].qp_handle = NULL; 1165 1166 RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags); 1167 1168 req.qp_cid = cpu_to_le32(qp->id); 1169 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 1170 (void *)&resp, NULL, 0); 1171 if (rc) { 1172 rcfw->qp_tbl[qp->id].qp_id = qp->id; 1173 rcfw->qp_tbl[qp->id].qp_handle = qp; 1174 return rc; 1175 } 1176 1177 /* Must walk the associated CQs to nullified the QP ptr */ 1178 spin_lock_irqsave(&qp->scq->hwq.lock, flags); 1179 1180 __clean_cq(qp->scq, (u64)(unsigned long)qp); 1181 1182 if (qp->rcq && qp->rcq != qp->scq) { 1183 spin_lock(&qp->rcq->hwq.lock); 1184 __clean_cq(qp->rcq, (u64)(unsigned long)qp); 1185 spin_unlock(&qp->rcq->hwq.lock); 1186 } 1187 1188 spin_unlock_irqrestore(&qp->scq->hwq.lock, flags); 1189 1190 bnxt_qplib_free_qp_hdr_buf(res, qp); 1191 bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq); 1192 kfree(qp->sq.swq); 1193 1194 bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq); 1195 kfree(qp->rq.swq); 1196 1197 if (qp->irrq.max_elements) 1198 bnxt_qplib_free_hwq(res->pdev, &qp->irrq); 1199 if (qp->orrq.max_elements) 1200 bnxt_qplib_free_hwq(res->pdev, &qp->orrq); 1201 1202 return 0; 1203 } 1204 1205 void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, 1206 struct bnxt_qplib_sge *sge) 1207 { 1208 struct bnxt_qplib_q *sq = &qp->sq; 1209 u32 sw_prod; 1210 1211 memset(sge, 0, sizeof(*sge)); 1212 1213 if (qp->sq_hdr_buf) { 1214 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); 1215 sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map + 1216 sw_prod * qp->sq_hdr_buf_size); 1217 sge->lkey = 0xFFFFFFFF; 1218 sge->size = qp->sq_hdr_buf_size; 1219 return qp->sq_hdr_buf + sw_prod * sge->size; 1220 } 1221 return NULL; 1222 } 1223 1224 u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp) 1225 { 1226 struct bnxt_qplib_q *rq = &qp->rq; 1227 1228 return HWQ_CMP(rq->hwq.prod, &rq->hwq); 1229 } 1230 1231 dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index) 1232 { 1233 return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size); 1234 } 1235 1236 void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, 1237 struct bnxt_qplib_sge *sge) 1238 { 1239 struct bnxt_qplib_q *rq = &qp->rq; 1240 u32 sw_prod; 1241 1242 memset(sge, 0, sizeof(*sge)); 1243 1244 if (qp->rq_hdr_buf) { 1245 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); 1246 sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map + 1247 sw_prod * qp->rq_hdr_buf_size); 1248 sge->lkey = 0xFFFFFFFF; 1249 sge->size = qp->rq_hdr_buf_size; 1250 return qp->rq_hdr_buf + sw_prod * sge->size; 1251 } 1252 return NULL; 1253 } 1254 1255 void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp) 1256 { 1257 struct bnxt_qplib_q *sq = &qp->sq; 1258 struct dbr_dbr db_msg = { 0 }; 1259 u32 sw_prod; 1260 1261 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); 1262 1263 db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) & 1264 DBR_DBR_INDEX_MASK); 1265 db_msg.type_xid = 1266 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | 1267 DBR_DBR_TYPE_SQ); 1268 /* Flush all the WQE writes to HW */ 1269 wmb(); 1270 __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); 1271 } 1272 1273 int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, 1274 struct bnxt_qplib_swqe *wqe) 1275 { 1276 struct bnxt_qplib_q *sq = &qp->sq; 1277 struct bnxt_qplib_swq *swq; 1278 struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr; 1279 struct sq_sge *hw_sge; 1280 struct bnxt_qplib_nq_work *nq_work = NULL; 1281 bool sch_handler = false; 1282 u32 sw_prod; 1283 u8 wqe_size16; 1284 int i, rc = 0, data_len = 0, pkt_num = 0; 1285 __le32 temp32; 1286 1287 if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) { 1288 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { 1289 sch_handler = true; 1290 dev_dbg(&sq->hwq.pdev->dev, 1291 "%s Error QP. Scheduling for poll_cq\n", 1292 __func__); 1293 goto queue_err; 1294 } 1295 } 1296 1297 if (bnxt_qplib_queue_full(sq)) { 1298 dev_err(&sq->hwq.pdev->dev, 1299 "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x", 1300 sq->hwq.prod, sq->hwq.cons, sq->hwq.max_elements, 1301 sq->q_full_delta); 1302 rc = -ENOMEM; 1303 goto done; 1304 } 1305 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); 1306 swq = &sq->swq[sw_prod]; 1307 swq->wr_id = wqe->wr_id; 1308 swq->type = wqe->type; 1309 swq->flags = wqe->flags; 1310 if (qp->sig_type) 1311 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP; 1312 swq->start_psn = sq->psn & BTH_PSN_MASK; 1313 1314 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; 1315 hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)] 1316 [get_sqe_idx(sw_prod)]; 1317 1318 memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE); 1319 1320 if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) { 1321 /* Copy the inline data */ 1322 if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { 1323 dev_warn(&sq->hwq.pdev->dev, 1324 "QPLIB: Inline data length > 96 detected"); 1325 data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH; 1326 } else { 1327 data_len = wqe->inline_len; 1328 } 1329 memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len); 1330 wqe_size16 = (data_len + 15) >> 4; 1331 } else { 1332 for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data; 1333 i < wqe->num_sge; i++, hw_sge++) { 1334 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); 1335 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); 1336 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); 1337 data_len += wqe->sg_list[i].size; 1338 } 1339 /* Each SGE entry = 1 WQE size16 */ 1340 wqe_size16 = wqe->num_sge; 1341 /* HW requires wqe size has room for atleast one SGE even if 1342 * none was supplied by ULP 1343 */ 1344 if (!wqe->num_sge) 1345 wqe_size16++; 1346 } 1347 1348 /* Specifics */ 1349 switch (wqe->type) { 1350 case BNXT_QPLIB_SWQE_TYPE_SEND: 1351 if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) { 1352 /* Assemble info for Raw Ethertype QPs */ 1353 struct sq_send_raweth_qp1 *sqe = 1354 (struct sq_send_raweth_qp1 *)hw_sq_send_hdr; 1355 1356 sqe->wqe_type = wqe->type; 1357 sqe->flags = wqe->flags; 1358 sqe->wqe_size = wqe_size16 + 1359 ((offsetof(typeof(*sqe), data) + 15) >> 4); 1360 sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action); 1361 sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags); 1362 sqe->length = cpu_to_le32(data_len); 1363 sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta & 1364 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) << 1365 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT); 1366 1367 break; 1368 } 1369 /* fall thru */ 1370 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: 1371 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: 1372 { 1373 struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr; 1374 1375 sqe->wqe_type = wqe->type; 1376 sqe->flags = wqe->flags; 1377 sqe->wqe_size = wqe_size16 + 1378 ((offsetof(typeof(*sqe), data) + 15) >> 4); 1379 sqe->inv_key_or_imm_data = cpu_to_le32( 1380 wqe->send.inv_key); 1381 if (qp->type == CMDQ_CREATE_QP_TYPE_UD) { 1382 sqe->q_key = cpu_to_le32(wqe->send.q_key); 1383 sqe->dst_qp = cpu_to_le32( 1384 wqe->send.dst_qp & SQ_SEND_DST_QP_MASK); 1385 sqe->length = cpu_to_le32(data_len); 1386 sqe->avid = cpu_to_le32(wqe->send.avid & 1387 SQ_SEND_AVID_MASK); 1388 sq->psn = (sq->psn + 1) & BTH_PSN_MASK; 1389 } else { 1390 sqe->length = cpu_to_le32(data_len); 1391 sqe->dst_qp = 0; 1392 sqe->avid = 0; 1393 if (qp->mtu) 1394 pkt_num = (data_len + qp->mtu - 1) / qp->mtu; 1395 if (!pkt_num) 1396 pkt_num = 1; 1397 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; 1398 } 1399 break; 1400 } 1401 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: 1402 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: 1403 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: 1404 { 1405 struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr; 1406 1407 sqe->wqe_type = wqe->type; 1408 sqe->flags = wqe->flags; 1409 sqe->wqe_size = wqe_size16 + 1410 ((offsetof(typeof(*sqe), data) + 15) >> 4); 1411 sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key); 1412 sqe->length = cpu_to_le32((u32)data_len); 1413 sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va); 1414 sqe->remote_key = cpu_to_le32(wqe->rdma.r_key); 1415 if (qp->mtu) 1416 pkt_num = (data_len + qp->mtu - 1) / qp->mtu; 1417 if (!pkt_num) 1418 pkt_num = 1; 1419 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; 1420 break; 1421 } 1422 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: 1423 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: 1424 { 1425 struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr; 1426 1427 sqe->wqe_type = wqe->type; 1428 sqe->flags = wqe->flags; 1429 sqe->remote_key = cpu_to_le32(wqe->atomic.r_key); 1430 sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va); 1431 sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data); 1432 sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data); 1433 if (qp->mtu) 1434 pkt_num = (data_len + qp->mtu - 1) / qp->mtu; 1435 if (!pkt_num) 1436 pkt_num = 1; 1437 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; 1438 break; 1439 } 1440 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: 1441 { 1442 struct sq_localinvalidate *sqe = 1443 (struct sq_localinvalidate *)hw_sq_send_hdr; 1444 1445 sqe->wqe_type = wqe->type; 1446 sqe->flags = wqe->flags; 1447 sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key); 1448 1449 break; 1450 } 1451 case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR: 1452 { 1453 struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr; 1454 1455 sqe->wqe_type = wqe->type; 1456 sqe->flags = wqe->flags; 1457 sqe->access_cntl = wqe->frmr.access_cntl | 1458 SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; 1459 sqe->zero_based_page_size_log = 1460 (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) << 1461 SQ_FR_PMR_PAGE_SIZE_LOG_SFT | 1462 (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0); 1463 sqe->l_key = cpu_to_le32(wqe->frmr.l_key); 1464 temp32 = cpu_to_le32(wqe->frmr.length); 1465 memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length)); 1466 sqe->numlevels_pbl_page_size_log = 1467 ((wqe->frmr.pbl_pg_sz_log << 1468 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) & 1469 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) | 1470 ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) & 1471 SQ_FR_PMR_NUMLEVELS_MASK); 1472 1473 for (i = 0; i < wqe->frmr.page_list_len; i++) 1474 wqe->frmr.pbl_ptr[i] = cpu_to_le64( 1475 wqe->frmr.page_list[i] | 1476 PTU_PTE_VALID); 1477 sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr); 1478 sqe->va = cpu_to_le64(wqe->frmr.va); 1479 1480 break; 1481 } 1482 case BNXT_QPLIB_SWQE_TYPE_BIND_MW: 1483 { 1484 struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr; 1485 1486 sqe->wqe_type = wqe->type; 1487 sqe->flags = wqe->flags; 1488 sqe->access_cntl = wqe->bind.access_cntl; 1489 sqe->mw_type_zero_based = wqe->bind.mw_type | 1490 (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0); 1491 sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key); 1492 sqe->l_key = cpu_to_le32(wqe->bind.r_key); 1493 sqe->va = cpu_to_le64(wqe->bind.va); 1494 temp32 = cpu_to_le32(wqe->bind.length); 1495 memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length)); 1496 break; 1497 } 1498 default: 1499 /* Bad wqe, return error */ 1500 rc = -EINVAL; 1501 goto done; 1502 } 1503 swq->next_psn = sq->psn & BTH_PSN_MASK; 1504 if (swq->psn_search) { 1505 swq->psn_search->opcode_start_psn = cpu_to_le32( 1506 ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) & 1507 SQ_PSN_SEARCH_START_PSN_MASK) | 1508 ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) & 1509 SQ_PSN_SEARCH_OPCODE_MASK)); 1510 swq->psn_search->flags_next_psn = cpu_to_le32( 1511 ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) & 1512 SQ_PSN_SEARCH_NEXT_PSN_MASK)); 1513 } 1514 queue_err: 1515 if (sch_handler) { 1516 /* Store the ULP info in the software structures */ 1517 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); 1518 swq = &sq->swq[sw_prod]; 1519 swq->wr_id = wqe->wr_id; 1520 swq->type = wqe->type; 1521 swq->flags = wqe->flags; 1522 if (qp->sig_type) 1523 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP; 1524 swq->start_psn = sq->psn & BTH_PSN_MASK; 1525 } 1526 sq->hwq.prod++; 1527 qp->wqe_cnt++; 1528 1529 done: 1530 if (sch_handler) { 1531 nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC); 1532 if (nq_work) { 1533 nq_work->cq = qp->scq; 1534 nq_work->nq = qp->scq->nq; 1535 INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task); 1536 queue_work(qp->scq->nq->cqn_wq, &nq_work->work); 1537 } else { 1538 dev_err(&sq->hwq.pdev->dev, 1539 "QPLIB: FP: Failed to allocate SQ nq_work!"); 1540 rc = -ENOMEM; 1541 } 1542 } 1543 return rc; 1544 } 1545 1546 void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp) 1547 { 1548 struct bnxt_qplib_q *rq = &qp->rq; 1549 struct dbr_dbr db_msg = { 0 }; 1550 u32 sw_prod; 1551 1552 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); 1553 db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) & 1554 DBR_DBR_INDEX_MASK); 1555 db_msg.type_xid = 1556 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | 1557 DBR_DBR_TYPE_RQ); 1558 1559 /* Flush the writes to HW Rx WQE before the ringing Rx DB */ 1560 wmb(); 1561 __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); 1562 } 1563 1564 int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, 1565 struct bnxt_qplib_swqe *wqe) 1566 { 1567 struct bnxt_qplib_q *rq = &qp->rq; 1568 struct rq_wqe *rqe, **rqe_ptr; 1569 struct sq_sge *hw_sge; 1570 struct bnxt_qplib_nq_work *nq_work = NULL; 1571 bool sch_handler = false; 1572 u32 sw_prod; 1573 int i, rc = 0; 1574 1575 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { 1576 sch_handler = true; 1577 dev_dbg(&rq->hwq.pdev->dev, 1578 "%s Error QP. Scheduling for poll_cq\n", 1579 __func__); 1580 goto queue_err; 1581 } 1582 if (bnxt_qplib_queue_full(rq)) { 1583 dev_err(&rq->hwq.pdev->dev, 1584 "QPLIB: FP: QP (0x%x) RQ is full!", qp->id); 1585 rc = -EINVAL; 1586 goto done; 1587 } 1588 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); 1589 rq->swq[sw_prod].wr_id = wqe->wr_id; 1590 1591 rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr; 1592 rqe = &rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)]; 1593 1594 memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE); 1595 1596 /* Calculate wqe_size16 and data_len */ 1597 for (i = 0, hw_sge = (struct sq_sge *)rqe->data; 1598 i < wqe->num_sge; i++, hw_sge++) { 1599 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); 1600 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); 1601 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); 1602 } 1603 rqe->wqe_type = wqe->type; 1604 rqe->flags = wqe->flags; 1605 rqe->wqe_size = wqe->num_sge + 1606 ((offsetof(typeof(*rqe), data) + 15) >> 4); 1607 /* HW requires wqe size has room for atleast one SGE even if none 1608 * was supplied by ULP 1609 */ 1610 if (!wqe->num_sge) 1611 rqe->wqe_size++; 1612 1613 /* Supply the rqe->wr_id index to the wr_id_tbl for now */ 1614 rqe->wr_id[0] = cpu_to_le32(sw_prod); 1615 1616 queue_err: 1617 if (sch_handler) { 1618 /* Store the ULP info in the software structures */ 1619 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); 1620 rq->swq[sw_prod].wr_id = wqe->wr_id; 1621 } 1622 1623 rq->hwq.prod++; 1624 if (sch_handler) { 1625 nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC); 1626 if (nq_work) { 1627 nq_work->cq = qp->rcq; 1628 nq_work->nq = qp->rcq->nq; 1629 INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task); 1630 queue_work(qp->rcq->nq->cqn_wq, &nq_work->work); 1631 } else { 1632 dev_err(&rq->hwq.pdev->dev, 1633 "QPLIB: FP: Failed to allocate RQ nq_work!"); 1634 rc = -ENOMEM; 1635 } 1636 } 1637 done: 1638 return rc; 1639 } 1640 1641 /* CQ */ 1642 1643 /* Spinlock must be held */ 1644 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq) 1645 { 1646 struct dbr_dbr db_msg = { 0 }; 1647 1648 db_msg.type_xid = 1649 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | 1650 DBR_DBR_TYPE_CQ_ARMENA); 1651 /* Flush memory writes before enabling the CQ */ 1652 wmb(); 1653 __iowrite64_copy(cq->dbr_base, &db_msg, sizeof(db_msg) / sizeof(u64)); 1654 } 1655 1656 static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type) 1657 { 1658 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; 1659 struct dbr_dbr db_msg = { 0 }; 1660 u32 sw_cons; 1661 1662 /* Ring DB */ 1663 sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq); 1664 db_msg.index = cpu_to_le32((sw_cons << DBR_DBR_INDEX_SFT) & 1665 DBR_DBR_INDEX_MASK); 1666 db_msg.type_xid = 1667 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | 1668 arm_type); 1669 /* flush memory writes before arming the CQ */ 1670 wmb(); 1671 __iowrite64_copy(cq->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); 1672 } 1673 1674 int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) 1675 { 1676 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 1677 struct cmdq_create_cq req; 1678 struct creq_create_cq_resp resp; 1679 struct bnxt_qplib_pbl *pbl; 1680 u16 cmd_flags = 0; 1681 int rc; 1682 1683 cq->hwq.max_elements = cq->max_wqe; 1684 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, cq->sghead, 1685 cq->nmap, &cq->hwq.max_elements, 1686 BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0, 1687 PAGE_SIZE, HWQ_TYPE_QUEUE); 1688 if (rc) 1689 goto exit; 1690 1691 RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags); 1692 1693 if (!cq->dpi) { 1694 dev_err(&rcfw->pdev->dev, 1695 "QPLIB: FP: CREATE_CQ failed due to NULL DPI"); 1696 return -EINVAL; 1697 } 1698 req.dpi = cpu_to_le32(cq->dpi->dpi); 1699 req.cq_handle = cpu_to_le64(cq->cq_handle); 1700 1701 req.cq_size = cpu_to_le32(cq->hwq.max_elements); 1702 pbl = &cq->hwq.pbl[PBL_LVL_0]; 1703 req.pg_size_lvl = cpu_to_le32( 1704 ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) << 1705 CMDQ_CREATE_CQ_LVL_SFT) | 1706 (pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K : 1707 pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K : 1708 pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K : 1709 pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M : 1710 pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M : 1711 pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G : 1712 CMDQ_CREATE_CQ_PG_SIZE_PG_4K)); 1713 1714 req.pbl = cpu_to_le64(pbl->pg_map_arr[0]); 1715 1716 req.cq_fco_cnq_id = cpu_to_le32( 1717 (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) << 1718 CMDQ_CREATE_CQ_CNQ_ID_SFT); 1719 1720 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 1721 (void *)&resp, NULL, 0); 1722 if (rc) 1723 goto fail; 1724 1725 cq->id = le32_to_cpu(resp.xid); 1726 cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem; 1727 cq->period = BNXT_QPLIB_QUEUE_START_PERIOD; 1728 init_waitqueue_head(&cq->waitq); 1729 INIT_LIST_HEAD(&cq->sqf_head); 1730 INIT_LIST_HEAD(&cq->rqf_head); 1731 spin_lock_init(&cq->compl_lock); 1732 1733 bnxt_qplib_arm_cq_enable(cq); 1734 return 0; 1735 1736 fail: 1737 bnxt_qplib_free_hwq(res->pdev, &cq->hwq); 1738 exit: 1739 return rc; 1740 } 1741 1742 int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) 1743 { 1744 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 1745 struct cmdq_destroy_cq req; 1746 struct creq_destroy_cq_resp resp; 1747 u16 cmd_flags = 0; 1748 int rc; 1749 1750 RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags); 1751 1752 req.cq_cid = cpu_to_le32(cq->id); 1753 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 1754 (void *)&resp, NULL, 0); 1755 if (rc) 1756 return rc; 1757 bnxt_qplib_free_hwq(res->pdev, &cq->hwq); 1758 return 0; 1759 } 1760 1761 static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp, 1762 struct bnxt_qplib_cqe **pcqe, int *budget) 1763 { 1764 u32 sw_prod, sw_cons; 1765 struct bnxt_qplib_cqe *cqe; 1766 int rc = 0; 1767 1768 /* Now complete all outstanding SQEs with FLUSHED_ERR */ 1769 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); 1770 cqe = *pcqe; 1771 while (*budget) { 1772 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); 1773 if (sw_cons == sw_prod) { 1774 break; 1775 } 1776 /* Skip the FENCE WQE completions */ 1777 if (sq->swq[sw_cons].wr_id == BNXT_QPLIB_FENCE_WRID) { 1778 bnxt_qplib_cancel_phantom_processing(qp); 1779 goto skip_compl; 1780 } 1781 memset(cqe, 0, sizeof(*cqe)); 1782 cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR; 1783 cqe->opcode = CQ_BASE_CQE_TYPE_REQ; 1784 cqe->qp_handle = (u64)(unsigned long)qp; 1785 cqe->wr_id = sq->swq[sw_cons].wr_id; 1786 cqe->src_qp = qp->id; 1787 cqe->type = sq->swq[sw_cons].type; 1788 cqe++; 1789 (*budget)--; 1790 skip_compl: 1791 sq->hwq.cons++; 1792 } 1793 *pcqe = cqe; 1794 if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod) 1795 /* Out of budget */ 1796 rc = -EAGAIN; 1797 1798 return rc; 1799 } 1800 1801 static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp, 1802 struct bnxt_qplib_cqe **pcqe, int *budget) 1803 { 1804 struct bnxt_qplib_cqe *cqe; 1805 u32 sw_prod, sw_cons; 1806 int rc = 0; 1807 int opcode = 0; 1808 1809 switch (qp->type) { 1810 case CMDQ_CREATE_QP1_TYPE_GSI: 1811 opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1; 1812 break; 1813 case CMDQ_CREATE_QP_TYPE_RC: 1814 opcode = CQ_BASE_CQE_TYPE_RES_RC; 1815 break; 1816 case CMDQ_CREATE_QP_TYPE_UD: 1817 opcode = CQ_BASE_CQE_TYPE_RES_UD; 1818 break; 1819 } 1820 1821 /* Flush the rest of the RQ */ 1822 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); 1823 cqe = *pcqe; 1824 while (*budget) { 1825 sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq); 1826 if (sw_cons == sw_prod) 1827 break; 1828 memset(cqe, 0, sizeof(*cqe)); 1829 cqe->status = 1830 CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR; 1831 cqe->opcode = opcode; 1832 cqe->qp_handle = (unsigned long)qp; 1833 cqe->wr_id = rq->swq[sw_cons].wr_id; 1834 cqe++; 1835 (*budget)--; 1836 rq->hwq.cons++; 1837 } 1838 *pcqe = cqe; 1839 if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod) 1840 /* Out of budget */ 1841 rc = -EAGAIN; 1842 1843 return rc; 1844 } 1845 1846 void bnxt_qplib_mark_qp_error(void *qp_handle) 1847 { 1848 struct bnxt_qplib_qp *qp = qp_handle; 1849 1850 if (!qp) 1851 return; 1852 1853 /* Must block new posting of SQ and RQ */ 1854 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; 1855 bnxt_qplib_cancel_phantom_processing(qp); 1856 1857 /* Add qp to flush list of the CQ */ 1858 __bnxt_qplib_add_flush_qp(qp); 1859 } 1860 1861 /* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive) 1862 * CQE is track from sw_cq_cons to max_element but valid only if VALID=1 1863 */ 1864 static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq, 1865 u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons) 1866 { 1867 struct bnxt_qplib_q *sq = &qp->sq; 1868 struct bnxt_qplib_swq *swq; 1869 u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx; 1870 struct cq_base *peek_hwcqe, **peek_hw_cqe_ptr; 1871 struct cq_req *peek_req_hwcqe; 1872 struct bnxt_qplib_qp *peek_qp; 1873 struct bnxt_qplib_q *peek_sq; 1874 int i, rc = 0; 1875 1876 /* Normal mode */ 1877 /* Check for the psn_search marking before completing */ 1878 swq = &sq->swq[sw_sq_cons]; 1879 if (swq->psn_search && 1880 le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) { 1881 /* Unmark */ 1882 swq->psn_search->flags_next_psn = cpu_to_le32 1883 (le32_to_cpu(swq->psn_search->flags_next_psn) 1884 & ~0x80000000); 1885 dev_dbg(&cq->hwq.pdev->dev, 1886 "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n", 1887 cq_cons, qp->id, sw_sq_cons, cqe_sq_cons); 1888 sq->condition = true; 1889 sq->send_phantom = true; 1890 1891 /* TODO: Only ARM if the previous SQE is ARMALL */ 1892 bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ_ARMALL); 1893 1894 rc = -EAGAIN; 1895 goto out; 1896 } 1897 if (sq->condition) { 1898 /* Peek at the completions */ 1899 peek_raw_cq_cons = cq->hwq.cons; 1900 peek_sw_cq_cons = cq_cons; 1901 i = cq->hwq.max_elements; 1902 while (i--) { 1903 peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq); 1904 peek_hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr; 1905 peek_hwcqe = &peek_hw_cqe_ptr[CQE_PG(peek_sw_cq_cons)] 1906 [CQE_IDX(peek_sw_cq_cons)]; 1907 /* If the next hwcqe is VALID */ 1908 if (CQE_CMP_VALID(peek_hwcqe, peek_raw_cq_cons, 1909 cq->hwq.max_elements)) { 1910 /* 1911 * The valid test of the entry must be done first before 1912 * reading any further. 1913 */ 1914 dma_rmb(); 1915 /* If the next hwcqe is a REQ */ 1916 if ((peek_hwcqe->cqe_type_toggle & 1917 CQ_BASE_CQE_TYPE_MASK) == 1918 CQ_BASE_CQE_TYPE_REQ) { 1919 peek_req_hwcqe = (struct cq_req *) 1920 peek_hwcqe; 1921 peek_qp = (struct bnxt_qplib_qp *) 1922 ((unsigned long) 1923 le64_to_cpu 1924 (peek_req_hwcqe->qp_handle)); 1925 peek_sq = &peek_qp->sq; 1926 peek_sq_cons_idx = HWQ_CMP(le16_to_cpu( 1927 peek_req_hwcqe->sq_cons_idx) - 1 1928 , &sq->hwq); 1929 /* If the hwcqe's sq's wr_id matches */ 1930 if (peek_sq == sq && 1931 sq->swq[peek_sq_cons_idx].wr_id == 1932 BNXT_QPLIB_FENCE_WRID) { 1933 /* 1934 * Unbreak only if the phantom 1935 * comes back 1936 */ 1937 dev_dbg(&cq->hwq.pdev->dev, 1938 "FP:Got Phantom CQE"); 1939 sq->condition = false; 1940 sq->single = true; 1941 rc = 0; 1942 goto out; 1943 } 1944 } 1945 /* Valid but not the phantom, so keep looping */ 1946 } else { 1947 /* Not valid yet, just exit and wait */ 1948 rc = -EINVAL; 1949 goto out; 1950 } 1951 peek_sw_cq_cons++; 1952 peek_raw_cq_cons++; 1953 } 1954 dev_err(&cq->hwq.pdev->dev, 1955 "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x", 1956 cq_cons, qp->id, sw_sq_cons, cqe_sq_cons); 1957 rc = -EINVAL; 1958 } 1959 out: 1960 return rc; 1961 } 1962 1963 static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, 1964 struct cq_req *hwcqe, 1965 struct bnxt_qplib_cqe **pcqe, int *budget, 1966 u32 cq_cons, struct bnxt_qplib_qp **lib_qp) 1967 { 1968 struct bnxt_qplib_qp *qp; 1969 struct bnxt_qplib_q *sq; 1970 struct bnxt_qplib_cqe *cqe; 1971 u32 sw_sq_cons, cqe_sq_cons; 1972 struct bnxt_qplib_swq *swq; 1973 int rc = 0; 1974 1975 qp = (struct bnxt_qplib_qp *)((unsigned long) 1976 le64_to_cpu(hwcqe->qp_handle)); 1977 if (!qp) { 1978 dev_err(&cq->hwq.pdev->dev, 1979 "QPLIB: FP: Process Req qp is NULL"); 1980 return -EINVAL; 1981 } 1982 sq = &qp->sq; 1983 1984 cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq); 1985 if (cqe_sq_cons > sq->hwq.max_elements) { 1986 dev_err(&cq->hwq.pdev->dev, 1987 "QPLIB: FP: CQ Process req reported "); 1988 dev_err(&cq->hwq.pdev->dev, 1989 "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x", 1990 cqe_sq_cons, sq->hwq.max_elements); 1991 return -EINVAL; 1992 } 1993 1994 if (qp->sq.flushed) { 1995 dev_dbg(&cq->hwq.pdev->dev, 1996 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); 1997 goto done; 1998 } 1999 /* Require to walk the sq's swq to fabricate CQEs for all previously 2000 * signaled SWQEs due to CQE aggregation from the current sq cons 2001 * to the cqe_sq_cons 2002 */ 2003 cqe = *pcqe; 2004 while (*budget) { 2005 sw_sq_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); 2006 if (sw_sq_cons == cqe_sq_cons) 2007 /* Done */ 2008 break; 2009 2010 swq = &sq->swq[sw_sq_cons]; 2011 memset(cqe, 0, sizeof(*cqe)); 2012 cqe->opcode = CQ_BASE_CQE_TYPE_REQ; 2013 cqe->qp_handle = (u64)(unsigned long)qp; 2014 cqe->src_qp = qp->id; 2015 cqe->wr_id = swq->wr_id; 2016 if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID) 2017 goto skip; 2018 cqe->type = swq->type; 2019 2020 /* For the last CQE, check for status. For errors, regardless 2021 * of the request being signaled or not, it must complete with 2022 * the hwcqe error status 2023 */ 2024 if (HWQ_CMP((sw_sq_cons + 1), &sq->hwq) == cqe_sq_cons && 2025 hwcqe->status != CQ_REQ_STATUS_OK) { 2026 cqe->status = hwcqe->status; 2027 dev_err(&cq->hwq.pdev->dev, 2028 "QPLIB: FP: CQ Processed Req "); 2029 dev_err(&cq->hwq.pdev->dev, 2030 "QPLIB: wr_id[%d] = 0x%llx with status 0x%x", 2031 sw_sq_cons, cqe->wr_id, cqe->status); 2032 cqe++; 2033 (*budget)--; 2034 bnxt_qplib_lock_buddy_cq(qp, cq); 2035 bnxt_qplib_mark_qp_error(qp); 2036 bnxt_qplib_unlock_buddy_cq(qp, cq); 2037 } else { 2038 if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) { 2039 /* Before we complete, do WA 9060 */ 2040 if (do_wa9060(qp, cq, cq_cons, sw_sq_cons, 2041 cqe_sq_cons)) { 2042 *lib_qp = qp; 2043 goto out; 2044 } 2045 cqe->status = CQ_REQ_STATUS_OK; 2046 cqe++; 2047 (*budget)--; 2048 } 2049 } 2050 skip: 2051 sq->hwq.cons++; 2052 if (sq->single) 2053 break; 2054 } 2055 out: 2056 *pcqe = cqe; 2057 if (HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_sq_cons) { 2058 /* Out of budget */ 2059 rc = -EAGAIN; 2060 goto done; 2061 } 2062 /* 2063 * Back to normal completion mode only after it has completed all of 2064 * the WC for this CQE 2065 */ 2066 sq->single = false; 2067 done: 2068 return rc; 2069 } 2070 2071 static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq, 2072 struct cq_res_rc *hwcqe, 2073 struct bnxt_qplib_cqe **pcqe, 2074 int *budget) 2075 { 2076 struct bnxt_qplib_qp *qp; 2077 struct bnxt_qplib_q *rq; 2078 struct bnxt_qplib_cqe *cqe; 2079 u32 wr_id_idx; 2080 int rc = 0; 2081 2082 qp = (struct bnxt_qplib_qp *)((unsigned long) 2083 le64_to_cpu(hwcqe->qp_handle)); 2084 if (!qp) { 2085 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL"); 2086 return -EINVAL; 2087 } 2088 if (qp->rq.flushed) { 2089 dev_dbg(&cq->hwq.pdev->dev, 2090 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); 2091 goto done; 2092 } 2093 2094 cqe = *pcqe; 2095 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; 2096 cqe->length = le32_to_cpu(hwcqe->length); 2097 cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key); 2098 cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle); 2099 cqe->flags = le16_to_cpu(hwcqe->flags); 2100 cqe->status = hwcqe->status; 2101 cqe->qp_handle = (u64)(unsigned long)qp; 2102 2103 wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) & 2104 CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK; 2105 rq = &qp->rq; 2106 if (wr_id_idx > rq->hwq.max_elements) { 2107 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process RC "); 2108 dev_err(&cq->hwq.pdev->dev, 2109 "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x", 2110 wr_id_idx, rq->hwq.max_elements); 2111 return -EINVAL; 2112 } 2113 2114 cqe->wr_id = rq->swq[wr_id_idx].wr_id; 2115 cqe++; 2116 (*budget)--; 2117 rq->hwq.cons++; 2118 *pcqe = cqe; 2119 2120 if (hwcqe->status != CQ_RES_RC_STATUS_OK) { 2121 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; 2122 /* Add qp to flush list of the CQ */ 2123 bnxt_qplib_lock_buddy_cq(qp, cq); 2124 __bnxt_qplib_add_flush_qp(qp); 2125 bnxt_qplib_unlock_buddy_cq(qp, cq); 2126 } 2127 2128 done: 2129 return rc; 2130 } 2131 2132 static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq, 2133 struct cq_res_ud *hwcqe, 2134 struct bnxt_qplib_cqe **pcqe, 2135 int *budget) 2136 { 2137 struct bnxt_qplib_qp *qp; 2138 struct bnxt_qplib_q *rq; 2139 struct bnxt_qplib_cqe *cqe; 2140 u32 wr_id_idx; 2141 int rc = 0; 2142 2143 qp = (struct bnxt_qplib_qp *)((unsigned long) 2144 le64_to_cpu(hwcqe->qp_handle)); 2145 if (!qp) { 2146 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL"); 2147 return -EINVAL; 2148 } 2149 if (qp->rq.flushed) { 2150 dev_dbg(&cq->hwq.pdev->dev, 2151 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); 2152 goto done; 2153 } 2154 cqe = *pcqe; 2155 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; 2156 cqe->length = le32_to_cpu(hwcqe->length); 2157 cqe->invrkey = le32_to_cpu(hwcqe->imm_data); 2158 cqe->flags = le16_to_cpu(hwcqe->flags); 2159 cqe->status = hwcqe->status; 2160 cqe->qp_handle = (u64)(unsigned long)qp; 2161 memcpy(cqe->smac, hwcqe->src_mac, 6); 2162 wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) 2163 & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK; 2164 cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) | 2165 ((le32_to_cpu( 2166 hwcqe->src_qp_high_srq_or_rq_wr_id) & 2167 CQ_RES_UD_SRC_QP_HIGH_MASK) >> 8); 2168 2169 rq = &qp->rq; 2170 if (wr_id_idx > rq->hwq.max_elements) { 2171 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process UD "); 2172 dev_err(&cq->hwq.pdev->dev, 2173 "QPLIB: wr_id idx %#x exceeded RQ max %#x", 2174 wr_id_idx, rq->hwq.max_elements); 2175 return -EINVAL; 2176 } 2177 2178 cqe->wr_id = rq->swq[wr_id_idx].wr_id; 2179 cqe++; 2180 (*budget)--; 2181 rq->hwq.cons++; 2182 *pcqe = cqe; 2183 2184 if (hwcqe->status != CQ_RES_RC_STATUS_OK) { 2185 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; 2186 /* Add qp to flush list of the CQ */ 2187 bnxt_qplib_lock_buddy_cq(qp, cq); 2188 __bnxt_qplib_add_flush_qp(qp); 2189 bnxt_qplib_unlock_buddy_cq(qp, cq); 2190 } 2191 done: 2192 return rc; 2193 } 2194 2195 bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq) 2196 { 2197 struct cq_base *hw_cqe, **hw_cqe_ptr; 2198 unsigned long flags; 2199 u32 sw_cons, raw_cons; 2200 bool rc = true; 2201 2202 spin_lock_irqsave(&cq->hwq.lock, flags); 2203 raw_cons = cq->hwq.cons; 2204 sw_cons = HWQ_CMP(raw_cons, &cq->hwq); 2205 hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr; 2206 hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)]; 2207 2208 /* Check for Valid bit. If the CQE is valid, return false */ 2209 rc = !CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements); 2210 spin_unlock_irqrestore(&cq->hwq.lock, flags); 2211 return rc; 2212 } 2213 2214 static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq, 2215 struct cq_res_raweth_qp1 *hwcqe, 2216 struct bnxt_qplib_cqe **pcqe, 2217 int *budget) 2218 { 2219 struct bnxt_qplib_qp *qp; 2220 struct bnxt_qplib_q *rq; 2221 struct bnxt_qplib_cqe *cqe; 2222 u32 wr_id_idx; 2223 int rc = 0; 2224 2225 qp = (struct bnxt_qplib_qp *)((unsigned long) 2226 le64_to_cpu(hwcqe->qp_handle)); 2227 if (!qp) { 2228 dev_err(&cq->hwq.pdev->dev, 2229 "QPLIB: process_cq Raw/QP1 qp is NULL"); 2230 return -EINVAL; 2231 } 2232 if (qp->rq.flushed) { 2233 dev_dbg(&cq->hwq.pdev->dev, 2234 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); 2235 goto done; 2236 } 2237 cqe = *pcqe; 2238 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; 2239 cqe->flags = le16_to_cpu(hwcqe->flags); 2240 cqe->qp_handle = (u64)(unsigned long)qp; 2241 2242 wr_id_idx = 2243 le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id) 2244 & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK; 2245 cqe->src_qp = qp->id; 2246 if (qp->id == 1 && !cqe->length) { 2247 /* Add workaround for the length misdetection */ 2248 cqe->length = 296; 2249 } else { 2250 cqe->length = le16_to_cpu(hwcqe->length); 2251 } 2252 cqe->pkey_index = qp->pkey_index; 2253 memcpy(cqe->smac, qp->smac, 6); 2254 2255 cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags); 2256 cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2); 2257 cqe->raweth_qp1_metadata = le32_to_cpu(hwcqe->raweth_qp1_metadata); 2258 2259 rq = &qp->rq; 2260 if (wr_id_idx > rq->hwq.max_elements) { 2261 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id "); 2262 dev_err(&cq->hwq.pdev->dev, "QPLIB: ix 0x%x exceeded RQ max 0x%x", 2263 wr_id_idx, rq->hwq.max_elements); 2264 return -EINVAL; 2265 } 2266 2267 cqe->wr_id = rq->swq[wr_id_idx].wr_id; 2268 cqe++; 2269 (*budget)--; 2270 rq->hwq.cons++; 2271 *pcqe = cqe; 2272 2273 if (hwcqe->status != CQ_RES_RC_STATUS_OK) { 2274 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; 2275 /* Add qp to flush list of the CQ */ 2276 bnxt_qplib_lock_buddy_cq(qp, cq); 2277 __bnxt_qplib_add_flush_qp(qp); 2278 bnxt_qplib_unlock_buddy_cq(qp, cq); 2279 } 2280 2281 done: 2282 return rc; 2283 } 2284 2285 static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, 2286 struct cq_terminal *hwcqe, 2287 struct bnxt_qplib_cqe **pcqe, 2288 int *budget) 2289 { 2290 struct bnxt_qplib_qp *qp; 2291 struct bnxt_qplib_q *sq, *rq; 2292 struct bnxt_qplib_cqe *cqe; 2293 u32 sw_cons = 0, cqe_cons; 2294 int rc = 0; 2295 2296 /* Check the Status */ 2297 if (hwcqe->status != CQ_TERMINAL_STATUS_OK) 2298 dev_warn(&cq->hwq.pdev->dev, 2299 "QPLIB: FP: CQ Process Terminal Error status = 0x%x", 2300 hwcqe->status); 2301 2302 qp = (struct bnxt_qplib_qp *)((unsigned long) 2303 le64_to_cpu(hwcqe->qp_handle)); 2304 if (!qp) { 2305 dev_err(&cq->hwq.pdev->dev, 2306 "QPLIB: FP: CQ Process terminal qp is NULL"); 2307 return -EINVAL; 2308 } 2309 2310 /* Must block new posting of SQ and RQ */ 2311 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; 2312 2313 sq = &qp->sq; 2314 rq = &qp->rq; 2315 2316 cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx); 2317 if (cqe_cons == 0xFFFF) 2318 goto do_rq; 2319 2320 if (cqe_cons > sq->hwq.max_elements) { 2321 dev_err(&cq->hwq.pdev->dev, 2322 "QPLIB: FP: CQ Process terminal reported "); 2323 dev_err(&cq->hwq.pdev->dev, 2324 "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x", 2325 cqe_cons, sq->hwq.max_elements); 2326 goto do_rq; 2327 } 2328 2329 if (qp->sq.flushed) { 2330 dev_dbg(&cq->hwq.pdev->dev, 2331 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); 2332 goto sq_done; 2333 } 2334 2335 /* Terminal CQE can also include aggregated successful CQEs prior. 2336 * So we must complete all CQEs from the current sq's cons to the 2337 * cq_cons with status OK 2338 */ 2339 cqe = *pcqe; 2340 while (*budget) { 2341 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); 2342 if (sw_cons == cqe_cons) 2343 break; 2344 if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) { 2345 memset(cqe, 0, sizeof(*cqe)); 2346 cqe->status = CQ_REQ_STATUS_OK; 2347 cqe->opcode = CQ_BASE_CQE_TYPE_REQ; 2348 cqe->qp_handle = (u64)(unsigned long)qp; 2349 cqe->src_qp = qp->id; 2350 cqe->wr_id = sq->swq[sw_cons].wr_id; 2351 cqe->type = sq->swq[sw_cons].type; 2352 cqe++; 2353 (*budget)--; 2354 } 2355 sq->hwq.cons++; 2356 } 2357 *pcqe = cqe; 2358 if (!(*budget) && sw_cons != cqe_cons) { 2359 /* Out of budget */ 2360 rc = -EAGAIN; 2361 goto sq_done; 2362 } 2363 sq_done: 2364 if (rc) 2365 return rc; 2366 do_rq: 2367 cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx); 2368 if (cqe_cons == 0xFFFF) { 2369 goto done; 2370 } else if (cqe_cons > rq->hwq.max_elements) { 2371 dev_err(&cq->hwq.pdev->dev, 2372 "QPLIB: FP: CQ Processed terminal "); 2373 dev_err(&cq->hwq.pdev->dev, 2374 "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x", 2375 cqe_cons, rq->hwq.max_elements); 2376 goto done; 2377 } 2378 2379 if (qp->rq.flushed) { 2380 dev_dbg(&cq->hwq.pdev->dev, 2381 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); 2382 rc = 0; 2383 goto done; 2384 } 2385 2386 /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR 2387 * from the current rq->cons to the rq->prod regardless what the 2388 * rq->cons the terminal CQE indicates 2389 */ 2390 2391 /* Add qp to flush list of the CQ */ 2392 bnxt_qplib_lock_buddy_cq(qp, cq); 2393 __bnxt_qplib_add_flush_qp(qp); 2394 bnxt_qplib_unlock_buddy_cq(qp, cq); 2395 done: 2396 return rc; 2397 } 2398 2399 static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq, 2400 struct cq_cutoff *hwcqe) 2401 { 2402 /* Check the Status */ 2403 if (hwcqe->status != CQ_CUTOFF_STATUS_OK) { 2404 dev_err(&cq->hwq.pdev->dev, 2405 "QPLIB: FP: CQ Process Cutoff Error status = 0x%x", 2406 hwcqe->status); 2407 return -EINVAL; 2408 } 2409 clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); 2410 wake_up_interruptible(&cq->waitq); 2411 2412 return 0; 2413 } 2414 2415 int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq, 2416 struct bnxt_qplib_cqe *cqe, 2417 int num_cqes) 2418 { 2419 struct bnxt_qplib_qp *qp = NULL; 2420 u32 budget = num_cqes; 2421 unsigned long flags; 2422 2423 spin_lock_irqsave(&cq->hwq.lock, flags); 2424 list_for_each_entry(qp, &cq->sqf_head, sq_flush) { 2425 dev_dbg(&cq->hwq.pdev->dev, 2426 "QPLIB: FP: Flushing SQ QP= %p", 2427 qp); 2428 __flush_sq(&qp->sq, qp, &cqe, &budget); 2429 } 2430 2431 list_for_each_entry(qp, &cq->rqf_head, rq_flush) { 2432 dev_dbg(&cq->hwq.pdev->dev, 2433 "QPLIB: FP: Flushing RQ QP= %p", 2434 qp); 2435 __flush_rq(&qp->rq, qp, &cqe, &budget); 2436 } 2437 spin_unlock_irqrestore(&cq->hwq.lock, flags); 2438 2439 return num_cqes - budget; 2440 } 2441 2442 int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, 2443 int num_cqes, struct bnxt_qplib_qp **lib_qp) 2444 { 2445 struct cq_base *hw_cqe, **hw_cqe_ptr; 2446 unsigned long flags; 2447 u32 sw_cons, raw_cons; 2448 int budget, rc = 0; 2449 2450 spin_lock_irqsave(&cq->hwq.lock, flags); 2451 raw_cons = cq->hwq.cons; 2452 budget = num_cqes; 2453 2454 while (budget) { 2455 sw_cons = HWQ_CMP(raw_cons, &cq->hwq); 2456 hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr; 2457 hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)]; 2458 2459 /* Check for Valid bit */ 2460 if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements)) 2461 break; 2462 2463 /* 2464 * The valid test of the entry must be done first before 2465 * reading any further. 2466 */ 2467 dma_rmb(); 2468 /* From the device's respective CQE format to qplib_wc*/ 2469 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { 2470 case CQ_BASE_CQE_TYPE_REQ: 2471 rc = bnxt_qplib_cq_process_req(cq, 2472 (struct cq_req *)hw_cqe, 2473 &cqe, &budget, 2474 sw_cons, lib_qp); 2475 break; 2476 case CQ_BASE_CQE_TYPE_RES_RC: 2477 rc = bnxt_qplib_cq_process_res_rc(cq, 2478 (struct cq_res_rc *) 2479 hw_cqe, &cqe, 2480 &budget); 2481 break; 2482 case CQ_BASE_CQE_TYPE_RES_UD: 2483 rc = bnxt_qplib_cq_process_res_ud 2484 (cq, (struct cq_res_ud *)hw_cqe, &cqe, 2485 &budget); 2486 break; 2487 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: 2488 rc = bnxt_qplib_cq_process_res_raweth_qp1 2489 (cq, (struct cq_res_raweth_qp1 *) 2490 hw_cqe, &cqe, &budget); 2491 break; 2492 case CQ_BASE_CQE_TYPE_TERMINAL: 2493 rc = bnxt_qplib_cq_process_terminal 2494 (cq, (struct cq_terminal *)hw_cqe, 2495 &cqe, &budget); 2496 break; 2497 case CQ_BASE_CQE_TYPE_CUT_OFF: 2498 bnxt_qplib_cq_process_cutoff 2499 (cq, (struct cq_cutoff *)hw_cqe); 2500 /* Done processing this CQ */ 2501 goto exit; 2502 default: 2503 dev_err(&cq->hwq.pdev->dev, 2504 "QPLIB: process_cq unknown type 0x%lx", 2505 hw_cqe->cqe_type_toggle & 2506 CQ_BASE_CQE_TYPE_MASK); 2507 rc = -EINVAL; 2508 break; 2509 } 2510 if (rc < 0) { 2511 if (rc == -EAGAIN) 2512 break; 2513 /* Error while processing the CQE, just skip to the 2514 * next one 2515 */ 2516 dev_err(&cq->hwq.pdev->dev, 2517 "QPLIB: process_cqe error rc = 0x%x", rc); 2518 } 2519 raw_cons++; 2520 } 2521 if (cq->hwq.cons != raw_cons) { 2522 cq->hwq.cons = raw_cons; 2523 bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ); 2524 } 2525 exit: 2526 spin_unlock_irqrestore(&cq->hwq.lock, flags); 2527 return num_cqes - budget; 2528 } 2529 2530 void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type) 2531 { 2532 unsigned long flags; 2533 2534 spin_lock_irqsave(&cq->hwq.lock, flags); 2535 if (arm_type) 2536 bnxt_qplib_arm_cq(cq, arm_type); 2537 /* Using cq->arm_state variable to track whether to issue cq handler */ 2538 atomic_set(&cq->arm_state, 1); 2539 spin_unlock_irqrestore(&cq->hwq.lock, flags); 2540 } 2541 2542 void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp) 2543 { 2544 flush_workqueue(qp->scq->nq->cqn_wq); 2545 if (qp->scq != qp->rcq) 2546 flush_workqueue(qp->rcq->nq->cqn_wq); 2547 } 2548