1 /* 2 * Broadcom NetXtreme-E RoCE driver. 3 * 4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term 5 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the 22 * distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * Description: Fast Path Operators 37 */ 38 39 #define dev_fmt(fmt) "QPLIB: " fmt 40 41 #include <linux/interrupt.h> 42 #include <linux/spinlock.h> 43 #include <linux/sched.h> 44 #include <linux/slab.h> 45 #include <linux/pci.h> 46 #include <linux/delay.h> 47 #include <linux/prefetch.h> 48 #include <linux/if_ether.h> 49 50 #include "roce_hsi.h" 51 52 #include "qplib_res.h" 53 #include "qplib_rcfw.h" 54 #include "qplib_sp.h" 55 #include "qplib_fp.h" 56 57 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp); 58 59 static void bnxt_qplib_cancel_phantom_processing(struct bnxt_qplib_qp *qp) 60 { 61 qp->sq.condition = false; 62 qp->sq.send_phantom = false; 63 qp->sq.single = false; 64 } 65 66 /* Flush list */ 67 static void __bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp) 68 { 69 struct bnxt_qplib_cq *scq, *rcq; 70 71 scq = qp->scq; 72 rcq = qp->rcq; 73 74 if (!qp->sq.flushed) { 75 dev_dbg(&scq->hwq.pdev->dev, 76 "FP: Adding to SQ Flush list = %p\n", qp); 77 bnxt_qplib_cancel_phantom_processing(qp); 78 list_add_tail(&qp->sq_flush, &scq->sqf_head); 79 qp->sq.flushed = true; 80 } 81 if (!qp->srq) { 82 if (!qp->rq.flushed) { 83 dev_dbg(&rcq->hwq.pdev->dev, 84 "FP: Adding to RQ Flush list = %p\n", qp); 85 list_add_tail(&qp->rq_flush, &rcq->rqf_head); 86 qp->rq.flushed = true; 87 } 88 } 89 } 90 91 static void bnxt_qplib_acquire_cq_flush_locks(struct bnxt_qplib_qp *qp, 92 unsigned long *flags) 93 __acquires(&qp->scq->flush_lock) __acquires(&qp->rcq->flush_lock) 94 { 95 spin_lock_irqsave(&qp->scq->flush_lock, *flags); 96 if (qp->scq == qp->rcq) 97 __acquire(&qp->rcq->flush_lock); 98 else 99 spin_lock(&qp->rcq->flush_lock); 100 } 101 102 static void bnxt_qplib_release_cq_flush_locks(struct bnxt_qplib_qp *qp, 103 unsigned long *flags) 104 __releases(&qp->scq->flush_lock) __releases(&qp->rcq->flush_lock) 105 { 106 if (qp->scq == qp->rcq) 107 __release(&qp->rcq->flush_lock); 108 else 109 spin_unlock(&qp->rcq->flush_lock); 110 spin_unlock_irqrestore(&qp->scq->flush_lock, *flags); 111 } 112 113 void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp) 114 { 115 unsigned long flags; 116 117 bnxt_qplib_acquire_cq_flush_locks(qp, &flags); 118 __bnxt_qplib_add_flush_qp(qp); 119 bnxt_qplib_release_cq_flush_locks(qp, &flags); 120 } 121 122 static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp) 123 { 124 if (qp->sq.flushed) { 125 qp->sq.flushed = false; 126 list_del(&qp->sq_flush); 127 } 128 if (!qp->srq) { 129 if (qp->rq.flushed) { 130 qp->rq.flushed = false; 131 list_del(&qp->rq_flush); 132 } 133 } 134 } 135 136 void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp) 137 { 138 unsigned long flags; 139 140 bnxt_qplib_acquire_cq_flush_locks(qp, &flags); 141 __clean_cq(qp->scq, (u64)(unsigned long)qp); 142 qp->sq.hwq.prod = 0; 143 qp->sq.hwq.cons = 0; 144 __clean_cq(qp->rcq, (u64)(unsigned long)qp); 145 qp->rq.hwq.prod = 0; 146 qp->rq.hwq.cons = 0; 147 148 __bnxt_qplib_del_flush_qp(qp); 149 bnxt_qplib_release_cq_flush_locks(qp, &flags); 150 } 151 152 static void bnxt_qpn_cqn_sched_task(struct work_struct *work) 153 { 154 struct bnxt_qplib_nq_work *nq_work = 155 container_of(work, struct bnxt_qplib_nq_work, work); 156 157 struct bnxt_qplib_cq *cq = nq_work->cq; 158 struct bnxt_qplib_nq *nq = nq_work->nq; 159 160 if (cq && nq) { 161 spin_lock_bh(&cq->compl_lock); 162 if (atomic_read(&cq->arm_state) && nq->cqn_handler) { 163 dev_dbg(&nq->pdev->dev, 164 "%s:Trigger cq = %p event nq = %p\n", 165 __func__, cq, nq); 166 nq->cqn_handler(nq, cq); 167 } 168 spin_unlock_bh(&cq->compl_lock); 169 } 170 kfree(nq_work); 171 } 172 173 static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res, 174 struct bnxt_qplib_qp *qp) 175 { 176 struct bnxt_qplib_q *rq = &qp->rq; 177 struct bnxt_qplib_q *sq = &qp->sq; 178 179 if (qp->rq_hdr_buf) 180 dma_free_coherent(&res->pdev->dev, 181 rq->hwq.max_elements * qp->rq_hdr_buf_size, 182 qp->rq_hdr_buf, qp->rq_hdr_buf_map); 183 if (qp->sq_hdr_buf) 184 dma_free_coherent(&res->pdev->dev, 185 sq->hwq.max_elements * qp->sq_hdr_buf_size, 186 qp->sq_hdr_buf, qp->sq_hdr_buf_map); 187 qp->rq_hdr_buf = NULL; 188 qp->sq_hdr_buf = NULL; 189 qp->rq_hdr_buf_map = 0; 190 qp->sq_hdr_buf_map = 0; 191 qp->sq_hdr_buf_size = 0; 192 qp->rq_hdr_buf_size = 0; 193 } 194 195 static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res, 196 struct bnxt_qplib_qp *qp) 197 { 198 struct bnxt_qplib_q *rq = &qp->rq; 199 struct bnxt_qplib_q *sq = &qp->sq; 200 int rc = 0; 201 202 if (qp->sq_hdr_buf_size && sq->hwq.max_elements) { 203 qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, 204 sq->hwq.max_elements * 205 qp->sq_hdr_buf_size, 206 &qp->sq_hdr_buf_map, GFP_KERNEL); 207 if (!qp->sq_hdr_buf) { 208 rc = -ENOMEM; 209 dev_err(&res->pdev->dev, 210 "Failed to create sq_hdr_buf\n"); 211 goto fail; 212 } 213 } 214 215 if (qp->rq_hdr_buf_size && rq->hwq.max_elements) { 216 qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, 217 rq->hwq.max_elements * 218 qp->rq_hdr_buf_size, 219 &qp->rq_hdr_buf_map, 220 GFP_KERNEL); 221 if (!qp->rq_hdr_buf) { 222 rc = -ENOMEM; 223 dev_err(&res->pdev->dev, 224 "Failed to create rq_hdr_buf\n"); 225 goto fail; 226 } 227 } 228 return 0; 229 230 fail: 231 bnxt_qplib_free_qp_hdr_buf(res, qp); 232 return rc; 233 } 234 235 static void clean_nq(struct bnxt_qplib_nq *nq, struct bnxt_qplib_cq *cq) 236 { 237 struct bnxt_qplib_hwq *hwq = &nq->hwq; 238 struct nq_base *nqe, **nq_ptr; 239 int budget = nq->budget; 240 u32 sw_cons, raw_cons; 241 uintptr_t q_handle; 242 u16 type; 243 244 spin_lock_bh(&hwq->lock); 245 /* Service the NQ until empty */ 246 raw_cons = hwq->cons; 247 while (budget--) { 248 sw_cons = HWQ_CMP(raw_cons, hwq); 249 nq_ptr = (struct nq_base **)hwq->pbl_ptr; 250 nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]; 251 if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements)) 252 break; 253 254 /* 255 * The valid test of the entry must be done first before 256 * reading any further. 257 */ 258 dma_rmb(); 259 260 type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK; 261 switch (type) { 262 case NQ_BASE_TYPE_CQ_NOTIFICATION: 263 { 264 struct nq_cn *nqcne = (struct nq_cn *)nqe; 265 266 q_handle = le32_to_cpu(nqcne->cq_handle_low); 267 q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) 268 << 32; 269 if ((unsigned long)cq == q_handle) { 270 nqcne->cq_handle_low = 0; 271 nqcne->cq_handle_high = 0; 272 cq->cnq_events++; 273 } 274 break; 275 } 276 default: 277 break; 278 } 279 raw_cons++; 280 } 281 spin_unlock_bh(&hwq->lock); 282 } 283 284 /* Wait for receiving all NQEs for this CQ and clean the NQEs associated with 285 * this CQ. 286 */ 287 static void __wait_for_all_nqes(struct bnxt_qplib_cq *cq, u16 cnq_events) 288 { 289 u32 retry_cnt = 100; 290 291 while (retry_cnt--) { 292 if (cnq_events == cq->cnq_events) 293 return; 294 usleep_range(50, 100); 295 clean_nq(cq->nq, cq); 296 } 297 } 298 299 static void bnxt_qplib_service_nq(unsigned long data) 300 { 301 struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data; 302 struct bnxt_qplib_hwq *hwq = &nq->hwq; 303 int num_srqne_processed = 0; 304 int num_cqne_processed = 0; 305 struct bnxt_qplib_cq *cq; 306 int budget = nq->budget; 307 u32 sw_cons, raw_cons; 308 struct nq_base *nqe; 309 uintptr_t q_handle; 310 u16 type; 311 312 spin_lock_bh(&hwq->lock); 313 /* Service the NQ until empty */ 314 raw_cons = hwq->cons; 315 while (budget--) { 316 sw_cons = HWQ_CMP(raw_cons, hwq); 317 nqe = bnxt_qplib_get_qe(hwq, sw_cons, NULL); 318 if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements)) 319 break; 320 321 /* 322 * The valid test of the entry must be done first before 323 * reading any further. 324 */ 325 dma_rmb(); 326 327 type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK; 328 switch (type) { 329 case NQ_BASE_TYPE_CQ_NOTIFICATION: 330 { 331 struct nq_cn *nqcne = (struct nq_cn *)nqe; 332 333 q_handle = le32_to_cpu(nqcne->cq_handle_low); 334 q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) 335 << 32; 336 cq = (struct bnxt_qplib_cq *)(unsigned long)q_handle; 337 if (!cq) 338 break; 339 bnxt_qplib_armen_db(&cq->dbinfo, 340 DBC_DBC_TYPE_CQ_ARMENA); 341 spin_lock_bh(&cq->compl_lock); 342 atomic_set(&cq->arm_state, 0); 343 if (!nq->cqn_handler(nq, (cq))) 344 num_cqne_processed++; 345 else 346 dev_warn(&nq->pdev->dev, 347 "cqn - type 0x%x not handled\n", type); 348 cq->cnq_events++; 349 spin_unlock_bh(&cq->compl_lock); 350 break; 351 } 352 case NQ_BASE_TYPE_SRQ_EVENT: 353 { 354 struct bnxt_qplib_srq *srq; 355 struct nq_srq_event *nqsrqe = 356 (struct nq_srq_event *)nqe; 357 358 q_handle = le32_to_cpu(nqsrqe->srq_handle_low); 359 q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high) 360 << 32; 361 srq = (struct bnxt_qplib_srq *)q_handle; 362 bnxt_qplib_armen_db(&srq->dbinfo, 363 DBC_DBC_TYPE_SRQ_ARMENA); 364 if (!nq->srqn_handler(nq, 365 (struct bnxt_qplib_srq *)q_handle, 366 nqsrqe->event)) 367 num_srqne_processed++; 368 else 369 dev_warn(&nq->pdev->dev, 370 "SRQ event 0x%x not handled\n", 371 nqsrqe->event); 372 break; 373 } 374 case NQ_BASE_TYPE_DBQ_EVENT: 375 break; 376 default: 377 dev_warn(&nq->pdev->dev, 378 "nqe with type = 0x%x not handled\n", type); 379 break; 380 } 381 raw_cons++; 382 } 383 if (hwq->cons != raw_cons) { 384 hwq->cons = raw_cons; 385 bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, nq->res->cctx, true); 386 } 387 spin_unlock_bh(&hwq->lock); 388 } 389 390 static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance) 391 { 392 struct bnxt_qplib_nq *nq = dev_instance; 393 struct bnxt_qplib_hwq *hwq = &nq->hwq; 394 u32 sw_cons; 395 396 /* Prefetch the NQ element */ 397 sw_cons = HWQ_CMP(hwq->cons, hwq); 398 prefetch(bnxt_qplib_get_qe(hwq, sw_cons, NULL)); 399 400 /* Fan out to CPU affinitized kthreads? */ 401 tasklet_schedule(&nq->nq_tasklet); 402 403 return IRQ_HANDLED; 404 } 405 406 void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill) 407 { 408 tasklet_disable(&nq->nq_tasklet); 409 /* Mask h/w interrupt */ 410 bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, nq->res->cctx, false); 411 /* Sync with last running IRQ handler */ 412 synchronize_irq(nq->msix_vec); 413 if (kill) 414 tasklet_kill(&nq->nq_tasklet); 415 if (nq->requested) { 416 irq_set_affinity_hint(nq->msix_vec, NULL); 417 free_irq(nq->msix_vec, nq); 418 nq->requested = false; 419 } 420 } 421 422 void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq) 423 { 424 if (nq->cqn_wq) { 425 destroy_workqueue(nq->cqn_wq); 426 nq->cqn_wq = NULL; 427 } 428 429 /* Make sure the HW is stopped! */ 430 bnxt_qplib_nq_stop_irq(nq, true); 431 432 if (nq->nq_db.reg.bar_reg) { 433 iounmap(nq->nq_db.reg.bar_reg); 434 nq->nq_db.reg.bar_reg = NULL; 435 } 436 437 nq->cqn_handler = NULL; 438 nq->srqn_handler = NULL; 439 nq->msix_vec = 0; 440 } 441 442 int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx, 443 int msix_vector, bool need_init) 444 { 445 int rc; 446 447 if (nq->requested) 448 return -EFAULT; 449 450 nq->msix_vec = msix_vector; 451 if (need_init) 452 tasklet_init(&nq->nq_tasklet, bnxt_qplib_service_nq, 453 (unsigned long)nq); 454 else 455 tasklet_enable(&nq->nq_tasklet); 456 457 snprintf(nq->name, sizeof(nq->name), "bnxt_qplib_nq-%d", nq_indx); 458 rc = request_irq(nq->msix_vec, bnxt_qplib_nq_irq, 0, nq->name, nq); 459 if (rc) 460 return rc; 461 462 cpumask_clear(&nq->mask); 463 cpumask_set_cpu(nq_indx, &nq->mask); 464 rc = irq_set_affinity_hint(nq->msix_vec, &nq->mask); 465 if (rc) { 466 dev_warn(&nq->pdev->dev, 467 "set affinity failed; vector: %d nq_idx: %d\n", 468 nq->msix_vec, nq_indx); 469 } 470 nq->requested = true; 471 bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, nq->res->cctx, true); 472 473 return rc; 474 } 475 476 static int bnxt_qplib_map_nq_db(struct bnxt_qplib_nq *nq, u32 reg_offt) 477 { 478 resource_size_t reg_base; 479 struct bnxt_qplib_nq_db *nq_db; 480 struct pci_dev *pdev; 481 int rc = 0; 482 483 pdev = nq->pdev; 484 nq_db = &nq->nq_db; 485 486 nq_db->reg.bar_id = NQ_CONS_PCI_BAR_REGION; 487 nq_db->reg.bar_base = pci_resource_start(pdev, nq_db->reg.bar_id); 488 if (!nq_db->reg.bar_base) { 489 dev_err(&pdev->dev, "QPLIB: NQ BAR region %d resc start is 0!", 490 nq_db->reg.bar_id); 491 rc = -ENOMEM; 492 goto fail; 493 } 494 495 reg_base = nq_db->reg.bar_base + reg_offt; 496 /* Unconditionally map 8 bytes to support 57500 series */ 497 nq_db->reg.len = 8; 498 nq_db->reg.bar_reg = ioremap(reg_base, nq_db->reg.len); 499 if (!nq_db->reg.bar_reg) { 500 dev_err(&pdev->dev, "QPLIB: NQ BAR region %d mapping failed", 501 nq_db->reg.bar_id); 502 rc = -ENOMEM; 503 goto fail; 504 } 505 506 nq_db->dbinfo.db = nq_db->reg.bar_reg; 507 nq_db->dbinfo.hwq = &nq->hwq; 508 nq_db->dbinfo.xid = nq->ring_id; 509 fail: 510 return rc; 511 } 512 513 int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq, 514 int nq_idx, int msix_vector, int bar_reg_offset, 515 cqn_handler_t cqn_handler, 516 srqn_handler_t srqn_handler) 517 { 518 int rc = -1; 519 520 nq->pdev = pdev; 521 nq->cqn_handler = cqn_handler; 522 nq->srqn_handler = srqn_handler; 523 524 /* Have a task to schedule CQ notifiers in post send case */ 525 nq->cqn_wq = create_singlethread_workqueue("bnxt_qplib_nq"); 526 if (!nq->cqn_wq) 527 return -ENOMEM; 528 529 rc = bnxt_qplib_map_nq_db(nq, bar_reg_offset); 530 if (rc) 531 goto fail; 532 533 rc = bnxt_qplib_nq_start_irq(nq, nq_idx, msix_vector, true); 534 if (rc) { 535 dev_err(&nq->pdev->dev, 536 "Failed to request irq for nq-idx %d\n", nq_idx); 537 goto fail; 538 } 539 540 return 0; 541 fail: 542 bnxt_qplib_disable_nq(nq); 543 return rc; 544 } 545 546 void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq) 547 { 548 if (nq->hwq.max_elements) { 549 bnxt_qplib_free_hwq(nq->res, &nq->hwq); 550 nq->hwq.max_elements = 0; 551 } 552 } 553 554 int bnxt_qplib_alloc_nq(struct bnxt_qplib_res *res, struct bnxt_qplib_nq *nq) 555 { 556 struct bnxt_qplib_hwq_attr hwq_attr = {}; 557 struct bnxt_qplib_sg_info sginfo = {}; 558 559 nq->pdev = res->pdev; 560 nq->res = res; 561 if (!nq->hwq.max_elements || 562 nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT) 563 nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT; 564 565 sginfo.pgsize = PAGE_SIZE; 566 sginfo.pgshft = PAGE_SHIFT; 567 hwq_attr.res = res; 568 hwq_attr.sginfo = &sginfo; 569 hwq_attr.depth = nq->hwq.max_elements; 570 hwq_attr.stride = sizeof(struct nq_base); 571 hwq_attr.type = bnxt_qplib_get_hwq_type(nq->res); 572 if (bnxt_qplib_alloc_init_hwq(&nq->hwq, &hwq_attr)) { 573 dev_err(&nq->pdev->dev, "FP NQ allocation failed"); 574 return -ENOMEM; 575 } 576 nq->budget = 8; 577 return 0; 578 } 579 580 /* SRQ */ 581 void bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res, 582 struct bnxt_qplib_srq *srq) 583 { 584 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 585 struct cmdq_destroy_srq req; 586 struct creq_destroy_srq_resp resp; 587 u16 cmd_flags = 0; 588 int rc; 589 590 RCFW_CMD_PREP(req, DESTROY_SRQ, cmd_flags); 591 592 /* Configure the request */ 593 req.srq_cid = cpu_to_le32(srq->id); 594 595 rc = bnxt_qplib_rcfw_send_message(rcfw, (struct cmdq_base *)&req, 596 (struct creq_base *)&resp, NULL, 0); 597 kfree(srq->swq); 598 if (rc) 599 return; 600 bnxt_qplib_free_hwq(res, &srq->hwq); 601 } 602 603 int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, 604 struct bnxt_qplib_srq *srq) 605 { 606 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 607 struct bnxt_qplib_hwq_attr hwq_attr = {}; 608 struct creq_create_srq_resp resp; 609 struct cmdq_create_srq req; 610 struct bnxt_qplib_pbl *pbl; 611 u16 cmd_flags = 0; 612 u16 pg_sz_lvl; 613 int rc, idx; 614 615 hwq_attr.res = res; 616 hwq_attr.sginfo = &srq->sg_info; 617 hwq_attr.depth = srq->max_wqe; 618 hwq_attr.stride = srq->wqe_size; 619 hwq_attr.type = HWQ_TYPE_QUEUE; 620 rc = bnxt_qplib_alloc_init_hwq(&srq->hwq, &hwq_attr); 621 if (rc) 622 goto exit; 623 624 srq->swq = kcalloc(srq->hwq.max_elements, sizeof(*srq->swq), 625 GFP_KERNEL); 626 if (!srq->swq) { 627 rc = -ENOMEM; 628 goto fail; 629 } 630 631 RCFW_CMD_PREP(req, CREATE_SRQ, cmd_flags); 632 633 /* Configure the request */ 634 req.dpi = cpu_to_le32(srq->dpi->dpi); 635 req.srq_handle = cpu_to_le64((uintptr_t)srq); 636 637 req.srq_size = cpu_to_le16((u16)srq->hwq.max_elements); 638 pbl = &srq->hwq.pbl[PBL_LVL_0]; 639 pg_sz_lvl = ((u16)bnxt_qplib_base_pg_size(&srq->hwq) << 640 CMDQ_CREATE_SRQ_PG_SIZE_SFT); 641 pg_sz_lvl |= (srq->hwq.level & CMDQ_CREATE_SRQ_LVL_MASK) << 642 CMDQ_CREATE_SRQ_LVL_SFT; 643 req.pg_size_lvl = cpu_to_le16(pg_sz_lvl); 644 req.pbl = cpu_to_le64(pbl->pg_map_arr[0]); 645 req.pd_id = cpu_to_le32(srq->pd->id); 646 req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id); 647 648 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 649 (void *)&resp, NULL, 0); 650 if (rc) 651 goto fail; 652 653 spin_lock_init(&srq->lock); 654 srq->start_idx = 0; 655 srq->last_idx = srq->hwq.max_elements - 1; 656 for (idx = 0; idx < srq->hwq.max_elements; idx++) 657 srq->swq[idx].next_idx = idx + 1; 658 srq->swq[srq->last_idx].next_idx = -1; 659 660 srq->id = le32_to_cpu(resp.xid); 661 srq->dbinfo.hwq = &srq->hwq; 662 srq->dbinfo.xid = srq->id; 663 srq->dbinfo.db = srq->dpi->dbr; 664 srq->dbinfo.priv_db = res->dpi_tbl.dbr_bar_reg_iomem; 665 if (srq->threshold) 666 bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA); 667 srq->arm_req = false; 668 669 return 0; 670 fail: 671 bnxt_qplib_free_hwq(res, &srq->hwq); 672 kfree(srq->swq); 673 exit: 674 return rc; 675 } 676 677 int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res, 678 struct bnxt_qplib_srq *srq) 679 { 680 struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; 681 u32 sw_prod, sw_cons, count = 0; 682 683 sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq); 684 sw_cons = HWQ_CMP(srq_hwq->cons, srq_hwq); 685 686 count = sw_prod > sw_cons ? sw_prod - sw_cons : 687 srq_hwq->max_elements - sw_cons + sw_prod; 688 if (count > srq->threshold) { 689 srq->arm_req = false; 690 bnxt_qplib_srq_arm_db(&srq->dbinfo, srq->threshold); 691 } else { 692 /* Deferred arming */ 693 srq->arm_req = true; 694 } 695 696 return 0; 697 } 698 699 int bnxt_qplib_query_srq(struct bnxt_qplib_res *res, 700 struct bnxt_qplib_srq *srq) 701 { 702 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 703 struct cmdq_query_srq req; 704 struct creq_query_srq_resp resp; 705 struct bnxt_qplib_rcfw_sbuf *sbuf; 706 struct creq_query_srq_resp_sb *sb; 707 u16 cmd_flags = 0; 708 int rc = 0; 709 710 RCFW_CMD_PREP(req, QUERY_SRQ, cmd_flags); 711 req.srq_cid = cpu_to_le32(srq->id); 712 713 /* Configure the request */ 714 sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb)); 715 if (!sbuf) 716 return -ENOMEM; 717 sb = sbuf->sb; 718 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, 719 (void *)sbuf, 0); 720 srq->threshold = le16_to_cpu(sb->srq_limit); 721 bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf); 722 723 return rc; 724 } 725 726 int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq, 727 struct bnxt_qplib_swqe *wqe) 728 { 729 struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; 730 struct rq_wqe *srqe; 731 struct sq_sge *hw_sge; 732 u32 sw_prod, sw_cons, count = 0; 733 int i, rc = 0, next; 734 735 spin_lock(&srq_hwq->lock); 736 if (srq->start_idx == srq->last_idx) { 737 dev_err(&srq_hwq->pdev->dev, 738 "FP: SRQ (0x%x) is full!\n", srq->id); 739 rc = -EINVAL; 740 spin_unlock(&srq_hwq->lock); 741 goto done; 742 } 743 next = srq->start_idx; 744 srq->start_idx = srq->swq[next].next_idx; 745 spin_unlock(&srq_hwq->lock); 746 747 sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq); 748 srqe = bnxt_qplib_get_qe(srq_hwq, sw_prod, NULL); 749 memset(srqe, 0, srq->wqe_size); 750 /* Calculate wqe_size16 and data_len */ 751 for (i = 0, hw_sge = (struct sq_sge *)srqe->data; 752 i < wqe->num_sge; i++, hw_sge++) { 753 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); 754 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); 755 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); 756 } 757 srqe->wqe_type = wqe->type; 758 srqe->flags = wqe->flags; 759 srqe->wqe_size = wqe->num_sge + 760 ((offsetof(typeof(*srqe), data) + 15) >> 4); 761 srqe->wr_id[0] = cpu_to_le32((u32)next); 762 srq->swq[next].wr_id = wqe->wr_id; 763 764 srq_hwq->prod++; 765 766 spin_lock(&srq_hwq->lock); 767 sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq); 768 /* retaining srq_hwq->cons for this logic 769 * actually the lock is only required to 770 * read srq_hwq->cons. 771 */ 772 sw_cons = HWQ_CMP(srq_hwq->cons, srq_hwq); 773 count = sw_prod > sw_cons ? sw_prod - sw_cons : 774 srq_hwq->max_elements - sw_cons + sw_prod; 775 spin_unlock(&srq_hwq->lock); 776 /* Ring DB */ 777 bnxt_qplib_ring_prod_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ); 778 if (srq->arm_req == true && count > srq->threshold) { 779 srq->arm_req = false; 780 bnxt_qplib_srq_arm_db(&srq->dbinfo, srq->threshold); 781 } 782 done: 783 return rc; 784 } 785 786 /* QP */ 787 int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) 788 { 789 struct bnxt_qplib_hwq_attr hwq_attr = {}; 790 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 791 struct bnxt_qplib_q *sq = &qp->sq; 792 struct bnxt_qplib_q *rq = &qp->rq; 793 struct creq_create_qp1_resp resp; 794 struct cmdq_create_qp1 req; 795 struct bnxt_qplib_pbl *pbl; 796 u16 cmd_flags = 0; 797 u32 qp_flags = 0; 798 u8 pg_sz_lvl; 799 int rc; 800 801 RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags); 802 803 /* General */ 804 req.type = qp->type; 805 req.dpi = cpu_to_le32(qp->dpi->dpi); 806 req.qp_handle = cpu_to_le64(qp->qp_handle); 807 808 /* SQ */ 809 hwq_attr.res = res; 810 hwq_attr.sginfo = &sq->sg_info; 811 hwq_attr.depth = sq->max_wqe; 812 hwq_attr.stride = sq->wqe_size; 813 hwq_attr.type = HWQ_TYPE_QUEUE; 814 rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); 815 if (rc) 816 goto exit; 817 818 sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL); 819 if (!sq->swq) { 820 rc = -ENOMEM; 821 goto fail_sq; 822 } 823 pbl = &sq->hwq.pbl[PBL_LVL_0]; 824 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); 825 pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) << 826 CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT); 827 pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK); 828 req.sq_pg_size_sq_lvl = pg_sz_lvl; 829 830 if (qp->scq) 831 req.scq_cid = cpu_to_le32(qp->scq->id); 832 /* RQ */ 833 if (rq->max_wqe) { 834 hwq_attr.res = res; 835 hwq_attr.sginfo = &rq->sg_info; 836 hwq_attr.stride = rq->wqe_size; 837 hwq_attr.depth = qp->rq.max_wqe; 838 hwq_attr.type = HWQ_TYPE_QUEUE; 839 rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); 840 if (rc) 841 goto fail_sq; 842 843 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq), 844 GFP_KERNEL); 845 if (!rq->swq) { 846 rc = -ENOMEM; 847 goto fail_rq; 848 } 849 pbl = &rq->hwq.pbl[PBL_LVL_0]; 850 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); 851 pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) << 852 CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT); 853 pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK); 854 req.rq_pg_size_rq_lvl = pg_sz_lvl; 855 if (qp->rcq) 856 req.rcq_cid = cpu_to_le32(qp->rcq->id); 857 } 858 /* Header buffer - allow hdr_buf pass in */ 859 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); 860 if (rc) { 861 rc = -ENOMEM; 862 goto fail; 863 } 864 qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE; 865 req.qp_flags = cpu_to_le32(qp_flags); 866 req.sq_size = cpu_to_le32(sq->hwq.max_elements); 867 req.rq_size = cpu_to_le32(rq->hwq.max_elements); 868 869 req.sq_fwo_sq_sge = 870 cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) << 871 CMDQ_CREATE_QP1_SQ_SGE_SFT); 872 req.rq_fwo_rq_sge = 873 cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) << 874 CMDQ_CREATE_QP1_RQ_SGE_SFT); 875 876 req.pd_id = cpu_to_le32(qp->pd->id); 877 878 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 879 (void *)&resp, NULL, 0); 880 if (rc) 881 goto fail; 882 883 qp->id = le32_to_cpu(resp.xid); 884 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; 885 qp->cctx = res->cctx; 886 sq->dbinfo.hwq = &sq->hwq; 887 sq->dbinfo.xid = qp->id; 888 sq->dbinfo.db = qp->dpi->dbr; 889 if (rq->max_wqe) { 890 rq->dbinfo.hwq = &rq->hwq; 891 rq->dbinfo.xid = qp->id; 892 rq->dbinfo.db = qp->dpi->dbr; 893 } 894 rcfw->qp_tbl[qp->id].qp_id = qp->id; 895 rcfw->qp_tbl[qp->id].qp_handle = (void *)qp; 896 897 return 0; 898 899 fail: 900 bnxt_qplib_free_qp_hdr_buf(res, qp); 901 fail_rq: 902 bnxt_qplib_free_hwq(res, &rq->hwq); 903 kfree(rq->swq); 904 fail_sq: 905 bnxt_qplib_free_hwq(res, &sq->hwq); 906 kfree(sq->swq); 907 exit: 908 return rc; 909 } 910 911 static void bnxt_qplib_init_psn_ptr(struct bnxt_qplib_qp *qp, int size) 912 { 913 struct bnxt_qplib_hwq *hwq; 914 struct bnxt_qplib_q *sq; 915 u64 fpsne, psne, psn_pg; 916 u16 indx_pad = 0, indx; 917 u16 pg_num, pg_indx; 918 u64 *page; 919 920 sq = &qp->sq; 921 hwq = &sq->hwq; 922 923 fpsne = (u64)bnxt_qplib_get_qe(hwq, hwq->max_elements, &psn_pg); 924 if (!IS_ALIGNED(fpsne, PAGE_SIZE)) 925 indx_pad = ALIGN(fpsne, PAGE_SIZE) / size; 926 927 page = (u64 *)psn_pg; 928 for (indx = 0; indx < hwq->max_elements; indx++) { 929 pg_num = (indx + indx_pad) / (PAGE_SIZE / size); 930 pg_indx = (indx + indx_pad) % (PAGE_SIZE / size); 931 psne = page[pg_num] + pg_indx * size; 932 sq->swq[indx].psn_ext = (struct sq_psn_search_ext *)psne; 933 sq->swq[indx].psn_search = (struct sq_psn_search *)psne; 934 } 935 } 936 937 int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) 938 { 939 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 940 struct bnxt_qplib_hwq_attr hwq_attr = {}; 941 struct bnxt_qplib_sg_info sginfo = {}; 942 struct bnxt_qplib_q *sq = &qp->sq; 943 struct bnxt_qplib_q *rq = &qp->rq; 944 struct creq_create_qp_resp resp; 945 int rc, req_size, psn_sz = 0; 946 struct bnxt_qplib_hwq *xrrq; 947 u16 cmd_flags = 0, max_ssge; 948 struct bnxt_qplib_pbl *pbl; 949 struct cmdq_create_qp req; 950 u32 qp_flags = 0; 951 u8 pg_sz_lvl; 952 u16 max_rsge; 953 954 RCFW_CMD_PREP(req, CREATE_QP, cmd_flags); 955 956 /* General */ 957 req.type = qp->type; 958 req.dpi = cpu_to_le32(qp->dpi->dpi); 959 req.qp_handle = cpu_to_le64(qp->qp_handle); 960 961 /* SQ */ 962 if (qp->type == CMDQ_CREATE_QP_TYPE_RC) { 963 psn_sz = bnxt_qplib_is_chip_gen_p5(res->cctx) ? 964 sizeof(struct sq_psn_search_ext) : 965 sizeof(struct sq_psn_search); 966 } 967 968 hwq_attr.res = res; 969 hwq_attr.sginfo = &sq->sg_info; 970 hwq_attr.stride = sq->wqe_size; 971 hwq_attr.depth = sq->max_wqe; 972 hwq_attr.aux_stride = psn_sz; 973 hwq_attr.aux_depth = hwq_attr.depth; 974 hwq_attr.type = HWQ_TYPE_QUEUE; 975 rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); 976 if (rc) 977 goto exit; 978 979 sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL); 980 if (!sq->swq) { 981 rc = -ENOMEM; 982 goto fail_sq; 983 } 984 985 if (psn_sz) 986 bnxt_qplib_init_psn_ptr(qp, psn_sz); 987 988 pbl = &sq->hwq.pbl[PBL_LVL_0]; 989 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); 990 pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) << 991 CMDQ_CREATE_QP_SQ_PG_SIZE_SFT); 992 pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK); 993 req.sq_pg_size_sq_lvl = pg_sz_lvl; 994 995 if (qp->scq) 996 req.scq_cid = cpu_to_le32(qp->scq->id); 997 998 /* RQ */ 999 if (rq->max_wqe) { 1000 hwq_attr.res = res; 1001 hwq_attr.sginfo = &rq->sg_info; 1002 hwq_attr.stride = rq->wqe_size; 1003 hwq_attr.depth = rq->max_wqe; 1004 hwq_attr.aux_stride = 0; 1005 hwq_attr.aux_depth = 0; 1006 hwq_attr.type = HWQ_TYPE_QUEUE; 1007 rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); 1008 if (rc) 1009 goto fail_sq; 1010 1011 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq), 1012 GFP_KERNEL); 1013 if (!rq->swq) { 1014 rc = -ENOMEM; 1015 goto fail_rq; 1016 } 1017 pbl = &rq->hwq.pbl[PBL_LVL_0]; 1018 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); 1019 pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) << 1020 CMDQ_CREATE_QP_RQ_PG_SIZE_SFT); 1021 pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK); 1022 req.rq_pg_size_rq_lvl = pg_sz_lvl; 1023 } else { 1024 /* SRQ */ 1025 if (qp->srq) { 1026 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED; 1027 req.srq_cid = cpu_to_le32(qp->srq->id); 1028 } 1029 } 1030 1031 if (qp->rcq) 1032 req.rcq_cid = cpu_to_le32(qp->rcq->id); 1033 1034 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE; 1035 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED; 1036 if (qp->sig_type) 1037 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION; 1038 req.qp_flags = cpu_to_le32(qp_flags); 1039 1040 req.sq_size = cpu_to_le32(sq->hwq.max_elements); 1041 req.rq_size = cpu_to_le32(rq->hwq.max_elements); 1042 qp->sq_hdr_buf = NULL; 1043 qp->rq_hdr_buf = NULL; 1044 1045 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); 1046 if (rc) 1047 goto fail_rq; 1048 1049 /* CTRL-22434: Irrespective of the requested SGE count on the SQ 1050 * always create the QP with max send sges possible if the requested 1051 * inline size is greater than 0. 1052 */ 1053 max_ssge = qp->max_inline_data ? 6 : sq->max_sge; 1054 req.sq_fwo_sq_sge = cpu_to_le16( 1055 ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK) 1056 << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0); 1057 max_rsge = bnxt_qplib_is_chip_gen_p5(res->cctx) ? 6 : rq->max_sge; 1058 req.rq_fwo_rq_sge = cpu_to_le16( 1059 ((max_rsge & CMDQ_CREATE_QP_RQ_SGE_MASK) 1060 << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0); 1061 /* ORRQ and IRRQ */ 1062 if (psn_sz) { 1063 xrrq = &qp->orrq; 1064 xrrq->max_elements = 1065 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); 1066 req_size = xrrq->max_elements * 1067 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1; 1068 req_size &= ~(PAGE_SIZE - 1); 1069 sginfo.pgsize = req_size; 1070 sginfo.pgshft = PAGE_SHIFT; 1071 1072 hwq_attr.res = res; 1073 hwq_attr.sginfo = &sginfo; 1074 hwq_attr.depth = xrrq->max_elements; 1075 hwq_attr.stride = BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE; 1076 hwq_attr.aux_stride = 0; 1077 hwq_attr.aux_depth = 0; 1078 hwq_attr.type = HWQ_TYPE_CTX; 1079 rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr); 1080 if (rc) 1081 goto fail_buf_free; 1082 pbl = &xrrq->pbl[PBL_LVL_0]; 1083 req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]); 1084 1085 xrrq = &qp->irrq; 1086 xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS( 1087 qp->max_dest_rd_atomic); 1088 req_size = xrrq->max_elements * 1089 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1; 1090 req_size &= ~(PAGE_SIZE - 1); 1091 sginfo.pgsize = req_size; 1092 hwq_attr.depth = xrrq->max_elements; 1093 hwq_attr.stride = BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE; 1094 rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr); 1095 if (rc) 1096 goto fail_orrq; 1097 1098 pbl = &xrrq->pbl[PBL_LVL_0]; 1099 req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]); 1100 } 1101 req.pd_id = cpu_to_le32(qp->pd->id); 1102 1103 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 1104 (void *)&resp, NULL, 0); 1105 if (rc) 1106 goto fail; 1107 1108 qp->id = le32_to_cpu(resp.xid); 1109 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; 1110 INIT_LIST_HEAD(&qp->sq_flush); 1111 INIT_LIST_HEAD(&qp->rq_flush); 1112 qp->cctx = res->cctx; 1113 sq->dbinfo.hwq = &sq->hwq; 1114 sq->dbinfo.xid = qp->id; 1115 sq->dbinfo.db = qp->dpi->dbr; 1116 if (rq->max_wqe) { 1117 rq->dbinfo.hwq = &rq->hwq; 1118 rq->dbinfo.xid = qp->id; 1119 rq->dbinfo.db = qp->dpi->dbr; 1120 } 1121 rcfw->qp_tbl[qp->id].qp_id = qp->id; 1122 rcfw->qp_tbl[qp->id].qp_handle = (void *)qp; 1123 1124 return 0; 1125 1126 fail: 1127 if (qp->irrq.max_elements) 1128 bnxt_qplib_free_hwq(res, &qp->irrq); 1129 fail_orrq: 1130 if (qp->orrq.max_elements) 1131 bnxt_qplib_free_hwq(res, &qp->orrq); 1132 fail_buf_free: 1133 bnxt_qplib_free_qp_hdr_buf(res, qp); 1134 fail_rq: 1135 bnxt_qplib_free_hwq(res, &rq->hwq); 1136 kfree(rq->swq); 1137 fail_sq: 1138 bnxt_qplib_free_hwq(res, &sq->hwq); 1139 kfree(sq->swq); 1140 exit: 1141 return rc; 1142 } 1143 1144 static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp) 1145 { 1146 switch (qp->state) { 1147 case CMDQ_MODIFY_QP_NEW_STATE_RTR: 1148 /* INIT->RTR, configure the path_mtu to the default 1149 * 2048 if not being requested 1150 */ 1151 if (!(qp->modify_flags & 1152 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) { 1153 qp->modify_flags |= 1154 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; 1155 qp->path_mtu = 1156 CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; 1157 } 1158 qp->modify_flags &= 1159 ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; 1160 /* Bono FW require the max_dest_rd_atomic to be >= 1 */ 1161 if (qp->max_dest_rd_atomic < 1) 1162 qp->max_dest_rd_atomic = 1; 1163 qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC; 1164 /* Bono FW 20.6.5 requires SGID_INDEX configuration */ 1165 if (!(qp->modify_flags & 1166 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) { 1167 qp->modify_flags |= 1168 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX; 1169 qp->ah.sgid_index = 0; 1170 } 1171 break; 1172 default: 1173 break; 1174 } 1175 } 1176 1177 static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp *qp) 1178 { 1179 switch (qp->state) { 1180 case CMDQ_MODIFY_QP_NEW_STATE_RTS: 1181 /* Bono FW requires the max_rd_atomic to be >= 1 */ 1182 if (qp->max_rd_atomic < 1) 1183 qp->max_rd_atomic = 1; 1184 /* Bono FW does not allow PKEY_INDEX, 1185 * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT, 1186 * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN, 1187 * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID 1188 * modification 1189 */ 1190 qp->modify_flags &= 1191 ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY | 1192 CMDQ_MODIFY_QP_MODIFY_MASK_DGID | 1193 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | 1194 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | 1195 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | 1196 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | 1197 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | 1198 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU | 1199 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN | 1200 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER | 1201 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC | 1202 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID); 1203 break; 1204 default: 1205 break; 1206 } 1207 } 1208 1209 static void __filter_modify_flags(struct bnxt_qplib_qp *qp) 1210 { 1211 switch (qp->cur_qp_state) { 1212 case CMDQ_MODIFY_QP_NEW_STATE_RESET: 1213 break; 1214 case CMDQ_MODIFY_QP_NEW_STATE_INIT: 1215 __modify_flags_from_init_state(qp); 1216 break; 1217 case CMDQ_MODIFY_QP_NEW_STATE_RTR: 1218 __modify_flags_from_rtr_state(qp); 1219 break; 1220 case CMDQ_MODIFY_QP_NEW_STATE_RTS: 1221 break; 1222 case CMDQ_MODIFY_QP_NEW_STATE_SQD: 1223 break; 1224 case CMDQ_MODIFY_QP_NEW_STATE_SQE: 1225 break; 1226 case CMDQ_MODIFY_QP_NEW_STATE_ERR: 1227 break; 1228 default: 1229 break; 1230 } 1231 } 1232 1233 int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) 1234 { 1235 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 1236 struct cmdq_modify_qp req; 1237 struct creq_modify_qp_resp resp; 1238 u16 cmd_flags = 0, pkey; 1239 u32 temp32[4]; 1240 u32 bmask; 1241 int rc; 1242 1243 RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags); 1244 1245 /* Filter out the qp_attr_mask based on the state->new transition */ 1246 __filter_modify_flags(qp); 1247 bmask = qp->modify_flags; 1248 req.modify_mask = cpu_to_le32(qp->modify_flags); 1249 req.qp_cid = cpu_to_le32(qp->id); 1250 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) { 1251 req.network_type_en_sqd_async_notify_new_state = 1252 (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) | 1253 (qp->en_sqd_async_notify ? 1254 CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0); 1255 } 1256 req.network_type_en_sqd_async_notify_new_state |= qp->nw_type; 1257 1258 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS) 1259 req.access = qp->access; 1260 1261 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) { 1262 if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl, 1263 qp->pkey_index, &pkey)) 1264 req.pkey = cpu_to_le16(pkey); 1265 } 1266 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY) 1267 req.qkey = cpu_to_le32(qp->qkey); 1268 1269 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) { 1270 memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid)); 1271 req.dgid[0] = cpu_to_le32(temp32[0]); 1272 req.dgid[1] = cpu_to_le32(temp32[1]); 1273 req.dgid[2] = cpu_to_le32(temp32[2]); 1274 req.dgid[3] = cpu_to_le32(temp32[3]); 1275 } 1276 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL) 1277 req.flow_label = cpu_to_le32(qp->ah.flow_label); 1278 1279 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX) 1280 req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id 1281 [qp->ah.sgid_index]); 1282 1283 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT) 1284 req.hop_limit = qp->ah.hop_limit; 1285 1286 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS) 1287 req.traffic_class = qp->ah.traffic_class; 1288 1289 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC) 1290 memcpy(req.dest_mac, qp->ah.dmac, 6); 1291 1292 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU) 1293 req.path_mtu = qp->path_mtu; 1294 1295 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT) 1296 req.timeout = qp->timeout; 1297 1298 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT) 1299 req.retry_cnt = qp->retry_cnt; 1300 1301 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY) 1302 req.rnr_retry = qp->rnr_retry; 1303 1304 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER) 1305 req.min_rnr_timer = qp->min_rnr_timer; 1306 1307 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN) 1308 req.rq_psn = cpu_to_le32(qp->rq.psn); 1309 1310 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN) 1311 req.sq_psn = cpu_to_le32(qp->sq.psn); 1312 1313 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC) 1314 req.max_rd_atomic = 1315 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); 1316 1317 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC) 1318 req.max_dest_rd_atomic = 1319 IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic); 1320 1321 req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements); 1322 req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements); 1323 req.sq_sge = cpu_to_le16(qp->sq.max_sge); 1324 req.rq_sge = cpu_to_le16(qp->rq.max_sge); 1325 req.max_inline_data = cpu_to_le32(qp->max_inline_data); 1326 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID) 1327 req.dest_qp_id = cpu_to_le32(qp->dest_qpn); 1328 1329 req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id); 1330 1331 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 1332 (void *)&resp, NULL, 0); 1333 if (rc) 1334 return rc; 1335 qp->cur_qp_state = qp->state; 1336 return 0; 1337 } 1338 1339 int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) 1340 { 1341 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 1342 struct cmdq_query_qp req; 1343 struct creq_query_qp_resp resp; 1344 struct bnxt_qplib_rcfw_sbuf *sbuf; 1345 struct creq_query_qp_resp_sb *sb; 1346 u16 cmd_flags = 0; 1347 u32 temp32[4]; 1348 int i, rc = 0; 1349 1350 RCFW_CMD_PREP(req, QUERY_QP, cmd_flags); 1351 1352 sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb)); 1353 if (!sbuf) 1354 return -ENOMEM; 1355 sb = sbuf->sb; 1356 1357 req.qp_cid = cpu_to_le32(qp->id); 1358 req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS; 1359 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, 1360 (void *)sbuf, 0); 1361 if (rc) 1362 goto bail; 1363 /* Extract the context from the side buffer */ 1364 qp->state = sb->en_sqd_async_notify_state & 1365 CREQ_QUERY_QP_RESP_SB_STATE_MASK; 1366 qp->en_sqd_async_notify = sb->en_sqd_async_notify_state & 1367 CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ? 1368 true : false; 1369 qp->access = sb->access; 1370 qp->pkey_index = le16_to_cpu(sb->pkey); 1371 qp->qkey = le32_to_cpu(sb->qkey); 1372 1373 temp32[0] = le32_to_cpu(sb->dgid[0]); 1374 temp32[1] = le32_to_cpu(sb->dgid[1]); 1375 temp32[2] = le32_to_cpu(sb->dgid[2]); 1376 temp32[3] = le32_to_cpu(sb->dgid[3]); 1377 memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data)); 1378 1379 qp->ah.flow_label = le32_to_cpu(sb->flow_label); 1380 1381 qp->ah.sgid_index = 0; 1382 for (i = 0; i < res->sgid_tbl.max; i++) { 1383 if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) { 1384 qp->ah.sgid_index = i; 1385 break; 1386 } 1387 } 1388 if (i == res->sgid_tbl.max) 1389 dev_warn(&res->pdev->dev, "SGID not found??\n"); 1390 1391 qp->ah.hop_limit = sb->hop_limit; 1392 qp->ah.traffic_class = sb->traffic_class; 1393 memcpy(qp->ah.dmac, sb->dest_mac, 6); 1394 qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) & 1395 CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK) >> 1396 CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT; 1397 qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) & 1398 CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) >> 1399 CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT; 1400 qp->timeout = sb->timeout; 1401 qp->retry_cnt = sb->retry_cnt; 1402 qp->rnr_retry = sb->rnr_retry; 1403 qp->min_rnr_timer = sb->min_rnr_timer; 1404 qp->rq.psn = le32_to_cpu(sb->rq_psn); 1405 qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic); 1406 qp->sq.psn = le32_to_cpu(sb->sq_psn); 1407 qp->max_dest_rd_atomic = 1408 IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic); 1409 qp->sq.max_wqe = qp->sq.hwq.max_elements; 1410 qp->rq.max_wqe = qp->rq.hwq.max_elements; 1411 qp->sq.max_sge = le16_to_cpu(sb->sq_sge); 1412 qp->rq.max_sge = le16_to_cpu(sb->rq_sge); 1413 qp->max_inline_data = le32_to_cpu(sb->max_inline_data); 1414 qp->dest_qpn = le32_to_cpu(sb->dest_qp_id); 1415 memcpy(qp->smac, sb->src_mac, 6); 1416 qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id); 1417 bail: 1418 bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf); 1419 return rc; 1420 } 1421 1422 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp) 1423 { 1424 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; 1425 struct cq_base *hw_cqe; 1426 int i; 1427 1428 for (i = 0; i < cq_hwq->max_elements; i++) { 1429 hw_cqe = bnxt_qplib_get_qe(cq_hwq, i, NULL); 1430 if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements)) 1431 continue; 1432 /* 1433 * The valid test of the entry must be done first before 1434 * reading any further. 1435 */ 1436 dma_rmb(); 1437 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { 1438 case CQ_BASE_CQE_TYPE_REQ: 1439 case CQ_BASE_CQE_TYPE_TERMINAL: 1440 { 1441 struct cq_req *cqe = (struct cq_req *)hw_cqe; 1442 1443 if (qp == le64_to_cpu(cqe->qp_handle)) 1444 cqe->qp_handle = 0; 1445 break; 1446 } 1447 case CQ_BASE_CQE_TYPE_RES_RC: 1448 case CQ_BASE_CQE_TYPE_RES_UD: 1449 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: 1450 { 1451 struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe; 1452 1453 if (qp == le64_to_cpu(cqe->qp_handle)) 1454 cqe->qp_handle = 0; 1455 break; 1456 } 1457 default: 1458 break; 1459 } 1460 } 1461 } 1462 1463 int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, 1464 struct bnxt_qplib_qp *qp) 1465 { 1466 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 1467 struct cmdq_destroy_qp req; 1468 struct creq_destroy_qp_resp resp; 1469 u16 cmd_flags = 0; 1470 int rc; 1471 1472 rcfw->qp_tbl[qp->id].qp_id = BNXT_QPLIB_QP_ID_INVALID; 1473 rcfw->qp_tbl[qp->id].qp_handle = NULL; 1474 1475 RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags); 1476 1477 req.qp_cid = cpu_to_le32(qp->id); 1478 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 1479 (void *)&resp, NULL, 0); 1480 if (rc) { 1481 rcfw->qp_tbl[qp->id].qp_id = qp->id; 1482 rcfw->qp_tbl[qp->id].qp_handle = qp; 1483 return rc; 1484 } 1485 1486 return 0; 1487 } 1488 1489 void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res, 1490 struct bnxt_qplib_qp *qp) 1491 { 1492 bnxt_qplib_free_qp_hdr_buf(res, qp); 1493 bnxt_qplib_free_hwq(res, &qp->sq.hwq); 1494 kfree(qp->sq.swq); 1495 1496 bnxt_qplib_free_hwq(res, &qp->rq.hwq); 1497 kfree(qp->rq.swq); 1498 1499 if (qp->irrq.max_elements) 1500 bnxt_qplib_free_hwq(res, &qp->irrq); 1501 if (qp->orrq.max_elements) 1502 bnxt_qplib_free_hwq(res, &qp->orrq); 1503 1504 } 1505 1506 void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, 1507 struct bnxt_qplib_sge *sge) 1508 { 1509 struct bnxt_qplib_q *sq = &qp->sq; 1510 u32 sw_prod; 1511 1512 memset(sge, 0, sizeof(*sge)); 1513 1514 if (qp->sq_hdr_buf) { 1515 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); 1516 sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map + 1517 sw_prod * qp->sq_hdr_buf_size); 1518 sge->lkey = 0xFFFFFFFF; 1519 sge->size = qp->sq_hdr_buf_size; 1520 return qp->sq_hdr_buf + sw_prod * sge->size; 1521 } 1522 return NULL; 1523 } 1524 1525 u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp) 1526 { 1527 struct bnxt_qplib_q *rq = &qp->rq; 1528 1529 return HWQ_CMP(rq->hwq.prod, &rq->hwq); 1530 } 1531 1532 dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index) 1533 { 1534 return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size); 1535 } 1536 1537 void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, 1538 struct bnxt_qplib_sge *sge) 1539 { 1540 struct bnxt_qplib_q *rq = &qp->rq; 1541 u32 sw_prod; 1542 1543 memset(sge, 0, sizeof(*sge)); 1544 1545 if (qp->rq_hdr_buf) { 1546 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); 1547 sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map + 1548 sw_prod * qp->rq_hdr_buf_size); 1549 sge->lkey = 0xFFFFFFFF; 1550 sge->size = qp->rq_hdr_buf_size; 1551 return qp->rq_hdr_buf + sw_prod * sge->size; 1552 } 1553 return NULL; 1554 } 1555 1556 static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp, 1557 struct bnxt_qplib_swqe *wqe, 1558 struct bnxt_qplib_swq *swq) 1559 { 1560 struct sq_psn_search_ext *psns_ext; 1561 struct sq_psn_search *psns; 1562 u32 flg_npsn; 1563 u32 op_spsn; 1564 1565 psns = swq->psn_search; 1566 psns_ext = swq->psn_ext; 1567 1568 op_spsn = ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) & 1569 SQ_PSN_SEARCH_START_PSN_MASK); 1570 op_spsn |= ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) & 1571 SQ_PSN_SEARCH_OPCODE_MASK); 1572 flg_npsn = ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) & 1573 SQ_PSN_SEARCH_NEXT_PSN_MASK); 1574 1575 if (bnxt_qplib_is_chip_gen_p5(qp->cctx)) { 1576 psns_ext->opcode_start_psn = cpu_to_le32(op_spsn); 1577 psns_ext->flags_next_psn = cpu_to_le32(flg_npsn); 1578 } else { 1579 psns->opcode_start_psn = cpu_to_le32(op_spsn); 1580 psns->flags_next_psn = cpu_to_le32(flg_npsn); 1581 } 1582 } 1583 1584 void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp) 1585 { 1586 struct bnxt_qplib_q *sq = &qp->sq; 1587 1588 bnxt_qplib_ring_prod_db(&sq->dbinfo, DBC_DBC_TYPE_SQ); 1589 } 1590 1591 int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, 1592 struct bnxt_qplib_swqe *wqe) 1593 { 1594 struct bnxt_qplib_nq_work *nq_work = NULL; 1595 int i, rc = 0, data_len = 0, pkt_num = 0; 1596 struct bnxt_qplib_q *sq = &qp->sq; 1597 struct sq_send *hw_sq_send_hdr; 1598 struct bnxt_qplib_swq *swq; 1599 bool sch_handler = false; 1600 struct sq_sge *hw_sge; 1601 u8 wqe_size16; 1602 __le32 temp32; 1603 u32 sw_prod; 1604 1605 if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) { 1606 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { 1607 sch_handler = true; 1608 dev_dbg(&sq->hwq.pdev->dev, 1609 "%s Error QP. Scheduling for poll_cq\n", 1610 __func__); 1611 goto queue_err; 1612 } 1613 } 1614 1615 if (bnxt_qplib_queue_full(sq)) { 1616 dev_err(&sq->hwq.pdev->dev, 1617 "prod = %#x cons = %#x qdepth = %#x delta = %#x\n", 1618 sq->hwq.prod, sq->hwq.cons, sq->hwq.max_elements, 1619 sq->q_full_delta); 1620 rc = -ENOMEM; 1621 goto done; 1622 } 1623 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); 1624 swq = &sq->swq[sw_prod]; 1625 swq->wr_id = wqe->wr_id; 1626 swq->type = wqe->type; 1627 swq->flags = wqe->flags; 1628 if (qp->sig_type) 1629 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP; 1630 swq->start_psn = sq->psn & BTH_PSN_MASK; 1631 1632 hw_sq_send_hdr = bnxt_qplib_get_qe(&sq->hwq, sw_prod, NULL); 1633 memset(hw_sq_send_hdr, 0, sq->wqe_size); 1634 1635 if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) { 1636 /* Copy the inline data */ 1637 if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { 1638 dev_warn(&sq->hwq.pdev->dev, 1639 "Inline data length > 96 detected\n"); 1640 data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH; 1641 } else { 1642 data_len = wqe->inline_len; 1643 } 1644 memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len); 1645 wqe_size16 = (data_len + 15) >> 4; 1646 } else { 1647 for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data; 1648 i < wqe->num_sge; i++, hw_sge++) { 1649 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); 1650 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); 1651 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); 1652 data_len += wqe->sg_list[i].size; 1653 } 1654 /* Each SGE entry = 1 WQE size16 */ 1655 wqe_size16 = wqe->num_sge; 1656 /* HW requires wqe size has room for atleast one SGE even if 1657 * none was supplied by ULP 1658 */ 1659 if (!wqe->num_sge) 1660 wqe_size16++; 1661 } 1662 1663 /* Specifics */ 1664 switch (wqe->type) { 1665 case BNXT_QPLIB_SWQE_TYPE_SEND: 1666 if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) { 1667 /* Assemble info for Raw Ethertype QPs */ 1668 struct sq_send_raweth_qp1 *sqe = 1669 (struct sq_send_raweth_qp1 *)hw_sq_send_hdr; 1670 1671 sqe->wqe_type = wqe->type; 1672 sqe->flags = wqe->flags; 1673 sqe->wqe_size = wqe_size16 + 1674 ((offsetof(typeof(*sqe), data) + 15) >> 4); 1675 sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action); 1676 sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags); 1677 sqe->length = cpu_to_le32(data_len); 1678 sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta & 1679 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) << 1680 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT); 1681 1682 break; 1683 } 1684 /* fall thru */ 1685 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: 1686 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: 1687 { 1688 struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr; 1689 1690 sqe->wqe_type = wqe->type; 1691 sqe->flags = wqe->flags; 1692 sqe->wqe_size = wqe_size16 + 1693 ((offsetof(typeof(*sqe), data) + 15) >> 4); 1694 sqe->inv_key_or_imm_data = cpu_to_le32( 1695 wqe->send.inv_key); 1696 if (qp->type == CMDQ_CREATE_QP_TYPE_UD || 1697 qp->type == CMDQ_CREATE_QP_TYPE_GSI) { 1698 sqe->q_key = cpu_to_le32(wqe->send.q_key); 1699 sqe->dst_qp = cpu_to_le32( 1700 wqe->send.dst_qp & SQ_SEND_DST_QP_MASK); 1701 sqe->length = cpu_to_le32(data_len); 1702 sqe->avid = cpu_to_le32(wqe->send.avid & 1703 SQ_SEND_AVID_MASK); 1704 sq->psn = (sq->psn + 1) & BTH_PSN_MASK; 1705 } else { 1706 sqe->length = cpu_to_le32(data_len); 1707 sqe->dst_qp = 0; 1708 sqe->avid = 0; 1709 if (qp->mtu) 1710 pkt_num = (data_len + qp->mtu - 1) / qp->mtu; 1711 if (!pkt_num) 1712 pkt_num = 1; 1713 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; 1714 } 1715 break; 1716 } 1717 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: 1718 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: 1719 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: 1720 { 1721 struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr; 1722 1723 sqe->wqe_type = wqe->type; 1724 sqe->flags = wqe->flags; 1725 sqe->wqe_size = wqe_size16 + 1726 ((offsetof(typeof(*sqe), data) + 15) >> 4); 1727 sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key); 1728 sqe->length = cpu_to_le32((u32)data_len); 1729 sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va); 1730 sqe->remote_key = cpu_to_le32(wqe->rdma.r_key); 1731 if (qp->mtu) 1732 pkt_num = (data_len + qp->mtu - 1) / qp->mtu; 1733 if (!pkt_num) 1734 pkt_num = 1; 1735 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; 1736 break; 1737 } 1738 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: 1739 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: 1740 { 1741 struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr; 1742 1743 sqe->wqe_type = wqe->type; 1744 sqe->flags = wqe->flags; 1745 sqe->remote_key = cpu_to_le32(wqe->atomic.r_key); 1746 sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va); 1747 sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data); 1748 sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data); 1749 if (qp->mtu) 1750 pkt_num = (data_len + qp->mtu - 1) / qp->mtu; 1751 if (!pkt_num) 1752 pkt_num = 1; 1753 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; 1754 break; 1755 } 1756 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: 1757 { 1758 struct sq_localinvalidate *sqe = 1759 (struct sq_localinvalidate *)hw_sq_send_hdr; 1760 1761 sqe->wqe_type = wqe->type; 1762 sqe->flags = wqe->flags; 1763 sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key); 1764 1765 break; 1766 } 1767 case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR: 1768 { 1769 struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr; 1770 1771 sqe->wqe_type = wqe->type; 1772 sqe->flags = wqe->flags; 1773 sqe->access_cntl = wqe->frmr.access_cntl | 1774 SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; 1775 sqe->zero_based_page_size_log = 1776 (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) << 1777 SQ_FR_PMR_PAGE_SIZE_LOG_SFT | 1778 (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0); 1779 sqe->l_key = cpu_to_le32(wqe->frmr.l_key); 1780 temp32 = cpu_to_le32(wqe->frmr.length); 1781 memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length)); 1782 sqe->numlevels_pbl_page_size_log = 1783 ((wqe->frmr.pbl_pg_sz_log << 1784 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) & 1785 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) | 1786 ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) & 1787 SQ_FR_PMR_NUMLEVELS_MASK); 1788 1789 for (i = 0; i < wqe->frmr.page_list_len; i++) 1790 wqe->frmr.pbl_ptr[i] = cpu_to_le64( 1791 wqe->frmr.page_list[i] | 1792 PTU_PTE_VALID); 1793 sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr); 1794 sqe->va = cpu_to_le64(wqe->frmr.va); 1795 1796 break; 1797 } 1798 case BNXT_QPLIB_SWQE_TYPE_BIND_MW: 1799 { 1800 struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr; 1801 1802 sqe->wqe_type = wqe->type; 1803 sqe->flags = wqe->flags; 1804 sqe->access_cntl = wqe->bind.access_cntl; 1805 sqe->mw_type_zero_based = wqe->bind.mw_type | 1806 (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0); 1807 sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key); 1808 sqe->l_key = cpu_to_le32(wqe->bind.r_key); 1809 sqe->va = cpu_to_le64(wqe->bind.va); 1810 temp32 = cpu_to_le32(wqe->bind.length); 1811 memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length)); 1812 break; 1813 } 1814 default: 1815 /* Bad wqe, return error */ 1816 rc = -EINVAL; 1817 goto done; 1818 } 1819 swq->next_psn = sq->psn & BTH_PSN_MASK; 1820 if (qp->type == CMDQ_CREATE_QP_TYPE_RC) 1821 bnxt_qplib_fill_psn_search(qp, wqe, swq); 1822 queue_err: 1823 if (sch_handler) { 1824 /* Store the ULP info in the software structures */ 1825 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); 1826 swq = &sq->swq[sw_prod]; 1827 swq->wr_id = wqe->wr_id; 1828 swq->type = wqe->type; 1829 swq->flags = wqe->flags; 1830 if (qp->sig_type) 1831 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP; 1832 swq->start_psn = sq->psn & BTH_PSN_MASK; 1833 } 1834 sq->hwq.prod++; 1835 qp->wqe_cnt++; 1836 1837 done: 1838 if (sch_handler) { 1839 nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC); 1840 if (nq_work) { 1841 nq_work->cq = qp->scq; 1842 nq_work->nq = qp->scq->nq; 1843 INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task); 1844 queue_work(qp->scq->nq->cqn_wq, &nq_work->work); 1845 } else { 1846 dev_err(&sq->hwq.pdev->dev, 1847 "FP: Failed to allocate SQ nq_work!\n"); 1848 rc = -ENOMEM; 1849 } 1850 } 1851 return rc; 1852 } 1853 1854 void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp) 1855 { 1856 struct bnxt_qplib_q *rq = &qp->rq; 1857 1858 bnxt_qplib_ring_prod_db(&rq->dbinfo, DBC_DBC_TYPE_RQ); 1859 } 1860 1861 int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, 1862 struct bnxt_qplib_swqe *wqe) 1863 { 1864 struct bnxt_qplib_nq_work *nq_work = NULL; 1865 struct bnxt_qplib_q *rq = &qp->rq; 1866 bool sch_handler = false; 1867 struct sq_sge *hw_sge; 1868 struct rq_wqe *rqe; 1869 int i, rc = 0; 1870 u32 sw_prod; 1871 1872 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { 1873 sch_handler = true; 1874 dev_dbg(&rq->hwq.pdev->dev, 1875 "%s: Error QP. Scheduling for poll_cq\n", __func__); 1876 goto queue_err; 1877 } 1878 if (bnxt_qplib_queue_full(rq)) { 1879 dev_err(&rq->hwq.pdev->dev, 1880 "FP: QP (0x%x) RQ is full!\n", qp->id); 1881 rc = -EINVAL; 1882 goto done; 1883 } 1884 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); 1885 rq->swq[sw_prod].wr_id = wqe->wr_id; 1886 1887 rqe = bnxt_qplib_get_qe(&rq->hwq, sw_prod, NULL); 1888 memset(rqe, 0, rq->wqe_size); 1889 1890 /* Calculate wqe_size16 and data_len */ 1891 for (i = 0, hw_sge = (struct sq_sge *)rqe->data; 1892 i < wqe->num_sge; i++, hw_sge++) { 1893 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); 1894 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); 1895 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); 1896 } 1897 rqe->wqe_type = wqe->type; 1898 rqe->flags = wqe->flags; 1899 rqe->wqe_size = wqe->num_sge + 1900 ((offsetof(typeof(*rqe), data) + 15) >> 4); 1901 /* HW requires wqe size has room for atleast one SGE even if none 1902 * was supplied by ULP 1903 */ 1904 if (!wqe->num_sge) 1905 rqe->wqe_size++; 1906 1907 /* Supply the rqe->wr_id index to the wr_id_tbl for now */ 1908 rqe->wr_id[0] = cpu_to_le32(sw_prod); 1909 1910 queue_err: 1911 if (sch_handler) { 1912 /* Store the ULP info in the software structures */ 1913 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); 1914 rq->swq[sw_prod].wr_id = wqe->wr_id; 1915 } 1916 1917 rq->hwq.prod++; 1918 if (sch_handler) { 1919 nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC); 1920 if (nq_work) { 1921 nq_work->cq = qp->rcq; 1922 nq_work->nq = qp->rcq->nq; 1923 INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task); 1924 queue_work(qp->rcq->nq->cqn_wq, &nq_work->work); 1925 } else { 1926 dev_err(&rq->hwq.pdev->dev, 1927 "FP: Failed to allocate RQ nq_work!\n"); 1928 rc = -ENOMEM; 1929 } 1930 } 1931 done: 1932 return rc; 1933 } 1934 1935 /* CQ */ 1936 int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) 1937 { 1938 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 1939 struct bnxt_qplib_hwq_attr hwq_attr = {}; 1940 struct creq_create_cq_resp resp; 1941 struct bnxt_qplib_pbl *pbl; 1942 struct cmdq_create_cq req; 1943 u16 cmd_flags = 0; 1944 u32 pg_sz_lvl; 1945 int rc; 1946 1947 hwq_attr.res = res; 1948 hwq_attr.depth = cq->max_wqe; 1949 hwq_attr.stride = sizeof(struct cq_base); 1950 hwq_attr.type = HWQ_TYPE_QUEUE; 1951 hwq_attr.sginfo = &cq->sg_info; 1952 rc = bnxt_qplib_alloc_init_hwq(&cq->hwq, &hwq_attr); 1953 if (rc) 1954 goto exit; 1955 1956 RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags); 1957 1958 if (!cq->dpi) { 1959 dev_err(&rcfw->pdev->dev, 1960 "FP: CREATE_CQ failed due to NULL DPI\n"); 1961 return -EINVAL; 1962 } 1963 req.dpi = cpu_to_le32(cq->dpi->dpi); 1964 req.cq_handle = cpu_to_le64(cq->cq_handle); 1965 req.cq_size = cpu_to_le32(cq->hwq.max_elements); 1966 pbl = &cq->hwq.pbl[PBL_LVL_0]; 1967 pg_sz_lvl = (bnxt_qplib_base_pg_size(&cq->hwq) << 1968 CMDQ_CREATE_CQ_PG_SIZE_SFT); 1969 pg_sz_lvl |= (cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK); 1970 req.pg_size_lvl = cpu_to_le32(pg_sz_lvl); 1971 req.pbl = cpu_to_le64(pbl->pg_map_arr[0]); 1972 req.cq_fco_cnq_id = cpu_to_le32( 1973 (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) << 1974 CMDQ_CREATE_CQ_CNQ_ID_SFT); 1975 1976 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 1977 (void *)&resp, NULL, 0); 1978 if (rc) 1979 goto fail; 1980 1981 cq->id = le32_to_cpu(resp.xid); 1982 cq->period = BNXT_QPLIB_QUEUE_START_PERIOD; 1983 init_waitqueue_head(&cq->waitq); 1984 INIT_LIST_HEAD(&cq->sqf_head); 1985 INIT_LIST_HEAD(&cq->rqf_head); 1986 spin_lock_init(&cq->compl_lock); 1987 spin_lock_init(&cq->flush_lock); 1988 1989 cq->dbinfo.hwq = &cq->hwq; 1990 cq->dbinfo.xid = cq->id; 1991 cq->dbinfo.db = cq->dpi->dbr; 1992 cq->dbinfo.priv_db = res->dpi_tbl.dbr_bar_reg_iomem; 1993 1994 bnxt_qplib_armen_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMENA); 1995 1996 return 0; 1997 1998 fail: 1999 bnxt_qplib_free_hwq(res, &cq->hwq); 2000 exit: 2001 return rc; 2002 } 2003 2004 int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) 2005 { 2006 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 2007 struct cmdq_destroy_cq req; 2008 struct creq_destroy_cq_resp resp; 2009 u16 total_cnq_events; 2010 u16 cmd_flags = 0; 2011 int rc; 2012 2013 RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags); 2014 2015 req.cq_cid = cpu_to_le32(cq->id); 2016 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, 2017 (void *)&resp, NULL, 0); 2018 if (rc) 2019 return rc; 2020 total_cnq_events = le16_to_cpu(resp.total_cnq_events); 2021 __wait_for_all_nqes(cq, total_cnq_events); 2022 bnxt_qplib_free_hwq(res, &cq->hwq); 2023 return 0; 2024 } 2025 2026 static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp, 2027 struct bnxt_qplib_cqe **pcqe, int *budget) 2028 { 2029 u32 sw_prod, sw_cons; 2030 struct bnxt_qplib_cqe *cqe; 2031 int rc = 0; 2032 2033 /* Now complete all outstanding SQEs with FLUSHED_ERR */ 2034 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); 2035 cqe = *pcqe; 2036 while (*budget) { 2037 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); 2038 if (sw_cons == sw_prod) { 2039 break; 2040 } 2041 /* Skip the FENCE WQE completions */ 2042 if (sq->swq[sw_cons].wr_id == BNXT_QPLIB_FENCE_WRID) { 2043 bnxt_qplib_cancel_phantom_processing(qp); 2044 goto skip_compl; 2045 } 2046 memset(cqe, 0, sizeof(*cqe)); 2047 cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR; 2048 cqe->opcode = CQ_BASE_CQE_TYPE_REQ; 2049 cqe->qp_handle = (u64)(unsigned long)qp; 2050 cqe->wr_id = sq->swq[sw_cons].wr_id; 2051 cqe->src_qp = qp->id; 2052 cqe->type = sq->swq[sw_cons].type; 2053 cqe++; 2054 (*budget)--; 2055 skip_compl: 2056 sq->hwq.cons++; 2057 } 2058 *pcqe = cqe; 2059 if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod) 2060 /* Out of budget */ 2061 rc = -EAGAIN; 2062 2063 return rc; 2064 } 2065 2066 static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp, 2067 struct bnxt_qplib_cqe **pcqe, int *budget) 2068 { 2069 struct bnxt_qplib_cqe *cqe; 2070 u32 sw_prod, sw_cons; 2071 int rc = 0; 2072 int opcode = 0; 2073 2074 switch (qp->type) { 2075 case CMDQ_CREATE_QP1_TYPE_GSI: 2076 opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1; 2077 break; 2078 case CMDQ_CREATE_QP_TYPE_RC: 2079 opcode = CQ_BASE_CQE_TYPE_RES_RC; 2080 break; 2081 case CMDQ_CREATE_QP_TYPE_UD: 2082 case CMDQ_CREATE_QP_TYPE_GSI: 2083 opcode = CQ_BASE_CQE_TYPE_RES_UD; 2084 break; 2085 } 2086 2087 /* Flush the rest of the RQ */ 2088 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); 2089 cqe = *pcqe; 2090 while (*budget) { 2091 sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq); 2092 if (sw_cons == sw_prod) 2093 break; 2094 memset(cqe, 0, sizeof(*cqe)); 2095 cqe->status = 2096 CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR; 2097 cqe->opcode = opcode; 2098 cqe->qp_handle = (unsigned long)qp; 2099 cqe->wr_id = rq->swq[sw_cons].wr_id; 2100 cqe++; 2101 (*budget)--; 2102 rq->hwq.cons++; 2103 } 2104 *pcqe = cqe; 2105 if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod) 2106 /* Out of budget */ 2107 rc = -EAGAIN; 2108 2109 return rc; 2110 } 2111 2112 void bnxt_qplib_mark_qp_error(void *qp_handle) 2113 { 2114 struct bnxt_qplib_qp *qp = qp_handle; 2115 2116 if (!qp) 2117 return; 2118 2119 /* Must block new posting of SQ and RQ */ 2120 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; 2121 bnxt_qplib_cancel_phantom_processing(qp); 2122 } 2123 2124 /* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive) 2125 * CQE is track from sw_cq_cons to max_element but valid only if VALID=1 2126 */ 2127 static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq, 2128 u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons) 2129 { 2130 u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx; 2131 struct bnxt_qplib_q *sq = &qp->sq; 2132 struct cq_req *peek_req_hwcqe; 2133 struct bnxt_qplib_qp *peek_qp; 2134 struct bnxt_qplib_q *peek_sq; 2135 struct bnxt_qplib_swq *swq; 2136 struct cq_base *peek_hwcqe; 2137 int i, rc = 0; 2138 2139 /* Normal mode */ 2140 /* Check for the psn_search marking before completing */ 2141 swq = &sq->swq[sw_sq_cons]; 2142 if (swq->psn_search && 2143 le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) { 2144 /* Unmark */ 2145 swq->psn_search->flags_next_psn = cpu_to_le32 2146 (le32_to_cpu(swq->psn_search->flags_next_psn) 2147 & ~0x80000000); 2148 dev_dbg(&cq->hwq.pdev->dev, 2149 "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n", 2150 cq_cons, qp->id, sw_sq_cons, cqe_sq_cons); 2151 sq->condition = true; 2152 sq->send_phantom = true; 2153 2154 /* TODO: Only ARM if the previous SQE is ARMALL */ 2155 bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMALL); 2156 rc = -EAGAIN; 2157 goto out; 2158 } 2159 if (sq->condition) { 2160 /* Peek at the completions */ 2161 peek_raw_cq_cons = cq->hwq.cons; 2162 peek_sw_cq_cons = cq_cons; 2163 i = cq->hwq.max_elements; 2164 while (i--) { 2165 peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq); 2166 peek_hwcqe = bnxt_qplib_get_qe(&cq->hwq, 2167 peek_sw_cq_cons, NULL); 2168 /* If the next hwcqe is VALID */ 2169 if (CQE_CMP_VALID(peek_hwcqe, peek_raw_cq_cons, 2170 cq->hwq.max_elements)) { 2171 /* 2172 * The valid test of the entry must be done first before 2173 * reading any further. 2174 */ 2175 dma_rmb(); 2176 /* If the next hwcqe is a REQ */ 2177 if ((peek_hwcqe->cqe_type_toggle & 2178 CQ_BASE_CQE_TYPE_MASK) == 2179 CQ_BASE_CQE_TYPE_REQ) { 2180 peek_req_hwcqe = (struct cq_req *) 2181 peek_hwcqe; 2182 peek_qp = (struct bnxt_qplib_qp *) 2183 ((unsigned long) 2184 le64_to_cpu 2185 (peek_req_hwcqe->qp_handle)); 2186 peek_sq = &peek_qp->sq; 2187 peek_sq_cons_idx = HWQ_CMP(le16_to_cpu( 2188 peek_req_hwcqe->sq_cons_idx) - 1 2189 , &sq->hwq); 2190 /* If the hwcqe's sq's wr_id matches */ 2191 if (peek_sq == sq && 2192 sq->swq[peek_sq_cons_idx].wr_id == 2193 BNXT_QPLIB_FENCE_WRID) { 2194 /* 2195 * Unbreak only if the phantom 2196 * comes back 2197 */ 2198 dev_dbg(&cq->hwq.pdev->dev, 2199 "FP: Got Phantom CQE\n"); 2200 sq->condition = false; 2201 sq->single = true; 2202 rc = 0; 2203 goto out; 2204 } 2205 } 2206 /* Valid but not the phantom, so keep looping */ 2207 } else { 2208 /* Not valid yet, just exit and wait */ 2209 rc = -EINVAL; 2210 goto out; 2211 } 2212 peek_sw_cq_cons++; 2213 peek_raw_cq_cons++; 2214 } 2215 dev_err(&cq->hwq.pdev->dev, 2216 "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n", 2217 cq_cons, qp->id, sw_sq_cons, cqe_sq_cons); 2218 rc = -EINVAL; 2219 } 2220 out: 2221 return rc; 2222 } 2223 2224 static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, 2225 struct cq_req *hwcqe, 2226 struct bnxt_qplib_cqe **pcqe, int *budget, 2227 u32 cq_cons, struct bnxt_qplib_qp **lib_qp) 2228 { 2229 u32 sw_sq_cons, cqe_sq_cons; 2230 struct bnxt_qplib_swq *swq; 2231 struct bnxt_qplib_cqe *cqe; 2232 struct bnxt_qplib_qp *qp; 2233 struct bnxt_qplib_q *sq; 2234 int rc = 0; 2235 2236 qp = (struct bnxt_qplib_qp *)((unsigned long) 2237 le64_to_cpu(hwcqe->qp_handle)); 2238 if (!qp) { 2239 dev_err(&cq->hwq.pdev->dev, 2240 "FP: Process Req qp is NULL\n"); 2241 return -EINVAL; 2242 } 2243 sq = &qp->sq; 2244 2245 cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq); 2246 if (cqe_sq_cons > sq->hwq.max_elements) { 2247 dev_err(&cq->hwq.pdev->dev, 2248 "FP: CQ Process req reported sq_cons_idx 0x%x which exceeded max 0x%x\n", 2249 cqe_sq_cons, sq->hwq.max_elements); 2250 return -EINVAL; 2251 } 2252 2253 if (qp->sq.flushed) { 2254 dev_dbg(&cq->hwq.pdev->dev, 2255 "%s: QP in Flush QP = %p\n", __func__, qp); 2256 goto done; 2257 } 2258 /* Require to walk the sq's swq to fabricate CQEs for all previously 2259 * signaled SWQEs due to CQE aggregation from the current sq cons 2260 * to the cqe_sq_cons 2261 */ 2262 cqe = *pcqe; 2263 while (*budget) { 2264 sw_sq_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); 2265 if (sw_sq_cons == cqe_sq_cons) 2266 /* Done */ 2267 break; 2268 2269 swq = &sq->swq[sw_sq_cons]; 2270 memset(cqe, 0, sizeof(*cqe)); 2271 cqe->opcode = CQ_BASE_CQE_TYPE_REQ; 2272 cqe->qp_handle = (u64)(unsigned long)qp; 2273 cqe->src_qp = qp->id; 2274 cqe->wr_id = swq->wr_id; 2275 if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID) 2276 goto skip; 2277 cqe->type = swq->type; 2278 2279 /* For the last CQE, check for status. For errors, regardless 2280 * of the request being signaled or not, it must complete with 2281 * the hwcqe error status 2282 */ 2283 if (HWQ_CMP((sw_sq_cons + 1), &sq->hwq) == cqe_sq_cons && 2284 hwcqe->status != CQ_REQ_STATUS_OK) { 2285 cqe->status = hwcqe->status; 2286 dev_err(&cq->hwq.pdev->dev, 2287 "FP: CQ Processed Req wr_id[%d] = 0x%llx with status 0x%x\n", 2288 sw_sq_cons, cqe->wr_id, cqe->status); 2289 cqe++; 2290 (*budget)--; 2291 bnxt_qplib_mark_qp_error(qp); 2292 /* Add qp to flush list of the CQ */ 2293 bnxt_qplib_add_flush_qp(qp); 2294 } else { 2295 /* Before we complete, do WA 9060 */ 2296 if (do_wa9060(qp, cq, cq_cons, sw_sq_cons, 2297 cqe_sq_cons)) { 2298 *lib_qp = qp; 2299 goto out; 2300 } 2301 if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) { 2302 cqe->status = CQ_REQ_STATUS_OK; 2303 cqe++; 2304 (*budget)--; 2305 } 2306 } 2307 skip: 2308 sq->hwq.cons++; 2309 if (sq->single) 2310 break; 2311 } 2312 out: 2313 *pcqe = cqe; 2314 if (HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_sq_cons) { 2315 /* Out of budget */ 2316 rc = -EAGAIN; 2317 goto done; 2318 } 2319 /* 2320 * Back to normal completion mode only after it has completed all of 2321 * the WC for this CQE 2322 */ 2323 sq->single = false; 2324 done: 2325 return rc; 2326 } 2327 2328 static void bnxt_qplib_release_srqe(struct bnxt_qplib_srq *srq, u32 tag) 2329 { 2330 spin_lock(&srq->hwq.lock); 2331 srq->swq[srq->last_idx].next_idx = (int)tag; 2332 srq->last_idx = (int)tag; 2333 srq->swq[srq->last_idx].next_idx = -1; 2334 srq->hwq.cons++; /* Support for SRQE counter */ 2335 spin_unlock(&srq->hwq.lock); 2336 } 2337 2338 static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq, 2339 struct cq_res_rc *hwcqe, 2340 struct bnxt_qplib_cqe **pcqe, 2341 int *budget) 2342 { 2343 struct bnxt_qplib_srq *srq; 2344 struct bnxt_qplib_cqe *cqe; 2345 struct bnxt_qplib_qp *qp; 2346 struct bnxt_qplib_q *rq; 2347 u32 wr_id_idx; 2348 int rc = 0; 2349 2350 qp = (struct bnxt_qplib_qp *)((unsigned long) 2351 le64_to_cpu(hwcqe->qp_handle)); 2352 if (!qp) { 2353 dev_err(&cq->hwq.pdev->dev, "process_cq RC qp is NULL\n"); 2354 return -EINVAL; 2355 } 2356 if (qp->rq.flushed) { 2357 dev_dbg(&cq->hwq.pdev->dev, 2358 "%s: QP in Flush QP = %p\n", __func__, qp); 2359 goto done; 2360 } 2361 2362 cqe = *pcqe; 2363 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; 2364 cqe->length = le32_to_cpu(hwcqe->length); 2365 cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key); 2366 cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle); 2367 cqe->flags = le16_to_cpu(hwcqe->flags); 2368 cqe->status = hwcqe->status; 2369 cqe->qp_handle = (u64)(unsigned long)qp; 2370 2371 wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) & 2372 CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK; 2373 if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) { 2374 srq = qp->srq; 2375 if (!srq) 2376 return -EINVAL; 2377 if (wr_id_idx >= srq->hwq.max_elements) { 2378 dev_err(&cq->hwq.pdev->dev, 2379 "FP: CQ Process RC wr_id idx 0x%x exceeded SRQ max 0x%x\n", 2380 wr_id_idx, srq->hwq.max_elements); 2381 return -EINVAL; 2382 } 2383 cqe->wr_id = srq->swq[wr_id_idx].wr_id; 2384 bnxt_qplib_release_srqe(srq, wr_id_idx); 2385 cqe++; 2386 (*budget)--; 2387 *pcqe = cqe; 2388 } else { 2389 rq = &qp->rq; 2390 if (wr_id_idx >= rq->hwq.max_elements) { 2391 dev_err(&cq->hwq.pdev->dev, 2392 "FP: CQ Process RC wr_id idx 0x%x exceeded RQ max 0x%x\n", 2393 wr_id_idx, rq->hwq.max_elements); 2394 return -EINVAL; 2395 } 2396 cqe->wr_id = rq->swq[wr_id_idx].wr_id; 2397 cqe++; 2398 (*budget)--; 2399 rq->hwq.cons++; 2400 *pcqe = cqe; 2401 2402 if (hwcqe->status != CQ_RES_RC_STATUS_OK) { 2403 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; 2404 /* Add qp to flush list of the CQ */ 2405 bnxt_qplib_add_flush_qp(qp); 2406 } 2407 } 2408 2409 done: 2410 return rc; 2411 } 2412 2413 static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq, 2414 struct cq_res_ud *hwcqe, 2415 struct bnxt_qplib_cqe **pcqe, 2416 int *budget) 2417 { 2418 struct bnxt_qplib_srq *srq; 2419 struct bnxt_qplib_cqe *cqe; 2420 struct bnxt_qplib_qp *qp; 2421 struct bnxt_qplib_q *rq; 2422 u32 wr_id_idx; 2423 int rc = 0; 2424 2425 qp = (struct bnxt_qplib_qp *)((unsigned long) 2426 le64_to_cpu(hwcqe->qp_handle)); 2427 if (!qp) { 2428 dev_err(&cq->hwq.pdev->dev, "process_cq UD qp is NULL\n"); 2429 return -EINVAL; 2430 } 2431 if (qp->rq.flushed) { 2432 dev_dbg(&cq->hwq.pdev->dev, 2433 "%s: QP in Flush QP = %p\n", __func__, qp); 2434 goto done; 2435 } 2436 cqe = *pcqe; 2437 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; 2438 cqe->length = le16_to_cpu(hwcqe->length) & CQ_RES_UD_LENGTH_MASK; 2439 cqe->cfa_meta = le16_to_cpu(hwcqe->cfa_metadata); 2440 cqe->invrkey = le32_to_cpu(hwcqe->imm_data); 2441 cqe->flags = le16_to_cpu(hwcqe->flags); 2442 cqe->status = hwcqe->status; 2443 cqe->qp_handle = (u64)(unsigned long)qp; 2444 /*FIXME: Endianness fix needed for smace */ 2445 memcpy(cqe->smac, hwcqe->src_mac, ETH_ALEN); 2446 wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) 2447 & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK; 2448 cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) | 2449 ((le32_to_cpu( 2450 hwcqe->src_qp_high_srq_or_rq_wr_id) & 2451 CQ_RES_UD_SRC_QP_HIGH_MASK) >> 8); 2452 2453 if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) { 2454 srq = qp->srq; 2455 if (!srq) 2456 return -EINVAL; 2457 2458 if (wr_id_idx >= srq->hwq.max_elements) { 2459 dev_err(&cq->hwq.pdev->dev, 2460 "FP: CQ Process UD wr_id idx 0x%x exceeded SRQ max 0x%x\n", 2461 wr_id_idx, srq->hwq.max_elements); 2462 return -EINVAL; 2463 } 2464 cqe->wr_id = srq->swq[wr_id_idx].wr_id; 2465 bnxt_qplib_release_srqe(srq, wr_id_idx); 2466 cqe++; 2467 (*budget)--; 2468 *pcqe = cqe; 2469 } else { 2470 rq = &qp->rq; 2471 if (wr_id_idx >= rq->hwq.max_elements) { 2472 dev_err(&cq->hwq.pdev->dev, 2473 "FP: CQ Process UD wr_id idx 0x%x exceeded RQ max 0x%x\n", 2474 wr_id_idx, rq->hwq.max_elements); 2475 return -EINVAL; 2476 } 2477 2478 cqe->wr_id = rq->swq[wr_id_idx].wr_id; 2479 cqe++; 2480 (*budget)--; 2481 rq->hwq.cons++; 2482 *pcqe = cqe; 2483 2484 if (hwcqe->status != CQ_RES_RC_STATUS_OK) { 2485 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; 2486 /* Add qp to flush list of the CQ */ 2487 bnxt_qplib_add_flush_qp(qp); 2488 } 2489 } 2490 done: 2491 return rc; 2492 } 2493 2494 bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq) 2495 { 2496 struct cq_base *hw_cqe; 2497 u32 sw_cons, raw_cons; 2498 bool rc = true; 2499 2500 raw_cons = cq->hwq.cons; 2501 sw_cons = HWQ_CMP(raw_cons, &cq->hwq); 2502 hw_cqe = bnxt_qplib_get_qe(&cq->hwq, sw_cons, NULL); 2503 /* Check for Valid bit. If the CQE is valid, return false */ 2504 rc = !CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements); 2505 return rc; 2506 } 2507 2508 static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq, 2509 struct cq_res_raweth_qp1 *hwcqe, 2510 struct bnxt_qplib_cqe **pcqe, 2511 int *budget) 2512 { 2513 struct bnxt_qplib_qp *qp; 2514 struct bnxt_qplib_q *rq; 2515 struct bnxt_qplib_srq *srq; 2516 struct bnxt_qplib_cqe *cqe; 2517 u32 wr_id_idx; 2518 int rc = 0; 2519 2520 qp = (struct bnxt_qplib_qp *)((unsigned long) 2521 le64_to_cpu(hwcqe->qp_handle)); 2522 if (!qp) { 2523 dev_err(&cq->hwq.pdev->dev, "process_cq Raw/QP1 qp is NULL\n"); 2524 return -EINVAL; 2525 } 2526 if (qp->rq.flushed) { 2527 dev_dbg(&cq->hwq.pdev->dev, 2528 "%s: QP in Flush QP = %p\n", __func__, qp); 2529 goto done; 2530 } 2531 cqe = *pcqe; 2532 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; 2533 cqe->flags = le16_to_cpu(hwcqe->flags); 2534 cqe->qp_handle = (u64)(unsigned long)qp; 2535 2536 wr_id_idx = 2537 le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id) 2538 & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK; 2539 cqe->src_qp = qp->id; 2540 if (qp->id == 1 && !cqe->length) { 2541 /* Add workaround for the length misdetection */ 2542 cqe->length = 296; 2543 } else { 2544 cqe->length = le16_to_cpu(hwcqe->length); 2545 } 2546 cqe->pkey_index = qp->pkey_index; 2547 memcpy(cqe->smac, qp->smac, 6); 2548 2549 cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags); 2550 cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2); 2551 cqe->raweth_qp1_metadata = le32_to_cpu(hwcqe->raweth_qp1_metadata); 2552 2553 if (cqe->flags & CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ) { 2554 srq = qp->srq; 2555 if (!srq) { 2556 dev_err(&cq->hwq.pdev->dev, 2557 "FP: SRQ used but not defined??\n"); 2558 return -EINVAL; 2559 } 2560 if (wr_id_idx >= srq->hwq.max_elements) { 2561 dev_err(&cq->hwq.pdev->dev, 2562 "FP: CQ Process Raw/QP1 wr_id idx 0x%x exceeded SRQ max 0x%x\n", 2563 wr_id_idx, srq->hwq.max_elements); 2564 return -EINVAL; 2565 } 2566 cqe->wr_id = srq->swq[wr_id_idx].wr_id; 2567 bnxt_qplib_release_srqe(srq, wr_id_idx); 2568 cqe++; 2569 (*budget)--; 2570 *pcqe = cqe; 2571 } else { 2572 rq = &qp->rq; 2573 if (wr_id_idx >= rq->hwq.max_elements) { 2574 dev_err(&cq->hwq.pdev->dev, 2575 "FP: CQ Process Raw/QP1 RQ wr_id idx 0x%x exceeded RQ max 0x%x\n", 2576 wr_id_idx, rq->hwq.max_elements); 2577 return -EINVAL; 2578 } 2579 cqe->wr_id = rq->swq[wr_id_idx].wr_id; 2580 cqe++; 2581 (*budget)--; 2582 rq->hwq.cons++; 2583 *pcqe = cqe; 2584 2585 if (hwcqe->status != CQ_RES_RC_STATUS_OK) { 2586 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; 2587 /* Add qp to flush list of the CQ */ 2588 bnxt_qplib_add_flush_qp(qp); 2589 } 2590 } 2591 2592 done: 2593 return rc; 2594 } 2595 2596 static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, 2597 struct cq_terminal *hwcqe, 2598 struct bnxt_qplib_cqe **pcqe, 2599 int *budget) 2600 { 2601 struct bnxt_qplib_qp *qp; 2602 struct bnxt_qplib_q *sq, *rq; 2603 struct bnxt_qplib_cqe *cqe; 2604 u32 sw_cons = 0, cqe_cons; 2605 int rc = 0; 2606 2607 /* Check the Status */ 2608 if (hwcqe->status != CQ_TERMINAL_STATUS_OK) 2609 dev_warn(&cq->hwq.pdev->dev, 2610 "FP: CQ Process Terminal Error status = 0x%x\n", 2611 hwcqe->status); 2612 2613 qp = (struct bnxt_qplib_qp *)((unsigned long) 2614 le64_to_cpu(hwcqe->qp_handle)); 2615 if (!qp) { 2616 dev_err(&cq->hwq.pdev->dev, 2617 "FP: CQ Process terminal qp is NULL\n"); 2618 return -EINVAL; 2619 } 2620 2621 /* Must block new posting of SQ and RQ */ 2622 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; 2623 2624 sq = &qp->sq; 2625 rq = &qp->rq; 2626 2627 cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx); 2628 if (cqe_cons == 0xFFFF) 2629 goto do_rq; 2630 2631 if (cqe_cons > sq->hwq.max_elements) { 2632 dev_err(&cq->hwq.pdev->dev, 2633 "FP: CQ Process terminal reported sq_cons_idx 0x%x which exceeded max 0x%x\n", 2634 cqe_cons, sq->hwq.max_elements); 2635 goto do_rq; 2636 } 2637 2638 if (qp->sq.flushed) { 2639 dev_dbg(&cq->hwq.pdev->dev, 2640 "%s: QP in Flush QP = %p\n", __func__, qp); 2641 goto sq_done; 2642 } 2643 2644 /* Terminal CQE can also include aggregated successful CQEs prior. 2645 * So we must complete all CQEs from the current sq's cons to the 2646 * cq_cons with status OK 2647 */ 2648 cqe = *pcqe; 2649 while (*budget) { 2650 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); 2651 if (sw_cons == cqe_cons) 2652 break; 2653 if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) { 2654 memset(cqe, 0, sizeof(*cqe)); 2655 cqe->status = CQ_REQ_STATUS_OK; 2656 cqe->opcode = CQ_BASE_CQE_TYPE_REQ; 2657 cqe->qp_handle = (u64)(unsigned long)qp; 2658 cqe->src_qp = qp->id; 2659 cqe->wr_id = sq->swq[sw_cons].wr_id; 2660 cqe->type = sq->swq[sw_cons].type; 2661 cqe++; 2662 (*budget)--; 2663 } 2664 sq->hwq.cons++; 2665 } 2666 *pcqe = cqe; 2667 if (!(*budget) && sw_cons != cqe_cons) { 2668 /* Out of budget */ 2669 rc = -EAGAIN; 2670 goto sq_done; 2671 } 2672 sq_done: 2673 if (rc) 2674 return rc; 2675 do_rq: 2676 cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx); 2677 if (cqe_cons == 0xFFFF) { 2678 goto done; 2679 } else if (cqe_cons > rq->hwq.max_elements) { 2680 dev_err(&cq->hwq.pdev->dev, 2681 "FP: CQ Processed terminal reported rq_cons_idx 0x%x exceeds max 0x%x\n", 2682 cqe_cons, rq->hwq.max_elements); 2683 goto done; 2684 } 2685 2686 if (qp->rq.flushed) { 2687 dev_dbg(&cq->hwq.pdev->dev, 2688 "%s: QP in Flush QP = %p\n", __func__, qp); 2689 rc = 0; 2690 goto done; 2691 } 2692 2693 /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR 2694 * from the current rq->cons to the rq->prod regardless what the 2695 * rq->cons the terminal CQE indicates 2696 */ 2697 2698 /* Add qp to flush list of the CQ */ 2699 bnxt_qplib_add_flush_qp(qp); 2700 done: 2701 return rc; 2702 } 2703 2704 static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq, 2705 struct cq_cutoff *hwcqe) 2706 { 2707 /* Check the Status */ 2708 if (hwcqe->status != CQ_CUTOFF_STATUS_OK) { 2709 dev_err(&cq->hwq.pdev->dev, 2710 "FP: CQ Process Cutoff Error status = 0x%x\n", 2711 hwcqe->status); 2712 return -EINVAL; 2713 } 2714 clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); 2715 wake_up_interruptible(&cq->waitq); 2716 2717 return 0; 2718 } 2719 2720 int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq, 2721 struct bnxt_qplib_cqe *cqe, 2722 int num_cqes) 2723 { 2724 struct bnxt_qplib_qp *qp = NULL; 2725 u32 budget = num_cqes; 2726 unsigned long flags; 2727 2728 spin_lock_irqsave(&cq->flush_lock, flags); 2729 list_for_each_entry(qp, &cq->sqf_head, sq_flush) { 2730 dev_dbg(&cq->hwq.pdev->dev, "FP: Flushing SQ QP= %p\n", qp); 2731 __flush_sq(&qp->sq, qp, &cqe, &budget); 2732 } 2733 2734 list_for_each_entry(qp, &cq->rqf_head, rq_flush) { 2735 dev_dbg(&cq->hwq.pdev->dev, "FP: Flushing RQ QP= %p\n", qp); 2736 __flush_rq(&qp->rq, qp, &cqe, &budget); 2737 } 2738 spin_unlock_irqrestore(&cq->flush_lock, flags); 2739 2740 return num_cqes - budget; 2741 } 2742 2743 int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, 2744 int num_cqes, struct bnxt_qplib_qp **lib_qp) 2745 { 2746 struct cq_base *hw_cqe; 2747 u32 sw_cons, raw_cons; 2748 int budget, rc = 0; 2749 2750 raw_cons = cq->hwq.cons; 2751 budget = num_cqes; 2752 2753 while (budget) { 2754 sw_cons = HWQ_CMP(raw_cons, &cq->hwq); 2755 hw_cqe = bnxt_qplib_get_qe(&cq->hwq, sw_cons, NULL); 2756 2757 /* Check for Valid bit */ 2758 if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements)) 2759 break; 2760 2761 /* 2762 * The valid test of the entry must be done first before 2763 * reading any further. 2764 */ 2765 dma_rmb(); 2766 /* From the device's respective CQE format to qplib_wc*/ 2767 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { 2768 case CQ_BASE_CQE_TYPE_REQ: 2769 rc = bnxt_qplib_cq_process_req(cq, 2770 (struct cq_req *)hw_cqe, 2771 &cqe, &budget, 2772 sw_cons, lib_qp); 2773 break; 2774 case CQ_BASE_CQE_TYPE_RES_RC: 2775 rc = bnxt_qplib_cq_process_res_rc(cq, 2776 (struct cq_res_rc *) 2777 hw_cqe, &cqe, 2778 &budget); 2779 break; 2780 case CQ_BASE_CQE_TYPE_RES_UD: 2781 rc = bnxt_qplib_cq_process_res_ud 2782 (cq, (struct cq_res_ud *)hw_cqe, &cqe, 2783 &budget); 2784 break; 2785 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: 2786 rc = bnxt_qplib_cq_process_res_raweth_qp1 2787 (cq, (struct cq_res_raweth_qp1 *) 2788 hw_cqe, &cqe, &budget); 2789 break; 2790 case CQ_BASE_CQE_TYPE_TERMINAL: 2791 rc = bnxt_qplib_cq_process_terminal 2792 (cq, (struct cq_terminal *)hw_cqe, 2793 &cqe, &budget); 2794 break; 2795 case CQ_BASE_CQE_TYPE_CUT_OFF: 2796 bnxt_qplib_cq_process_cutoff 2797 (cq, (struct cq_cutoff *)hw_cqe); 2798 /* Done processing this CQ */ 2799 goto exit; 2800 default: 2801 dev_err(&cq->hwq.pdev->dev, 2802 "process_cq unknown type 0x%lx\n", 2803 hw_cqe->cqe_type_toggle & 2804 CQ_BASE_CQE_TYPE_MASK); 2805 rc = -EINVAL; 2806 break; 2807 } 2808 if (rc < 0) { 2809 if (rc == -EAGAIN) 2810 break; 2811 /* Error while processing the CQE, just skip to the 2812 * next one 2813 */ 2814 dev_err(&cq->hwq.pdev->dev, 2815 "process_cqe error rc = 0x%x\n", rc); 2816 } 2817 raw_cons++; 2818 } 2819 if (cq->hwq.cons != raw_cons) { 2820 cq->hwq.cons = raw_cons; 2821 bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ); 2822 } 2823 exit: 2824 return num_cqes - budget; 2825 } 2826 2827 void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type) 2828 { 2829 if (arm_type) 2830 bnxt_qplib_ring_db(&cq->dbinfo, arm_type); 2831 /* Using cq->arm_state variable to track whether to issue cq handler */ 2832 atomic_set(&cq->arm_state, 1); 2833 } 2834 2835 void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp) 2836 { 2837 flush_workqueue(qp->scq->nq->cqn_wq); 2838 if (qp->scq != qp->rcq) 2839 flush_workqueue(qp->rcq->nq->cqn_wq); 2840 } 2841