1 /* 2 * Broadcom NetXtreme-E RoCE driver. 3 * 4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term 5 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the 22 * distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * Description: RDMA Controller HW interface 37 */ 38 39 #define dev_fmt(fmt) "QPLIB: " fmt 40 41 #include <linux/interrupt.h> 42 #include <linux/spinlock.h> 43 #include <linux/pci.h> 44 #include <linux/prefetch.h> 45 #include <linux/delay.h> 46 47 #include "roce_hsi.h" 48 #include "qplib_res.h" 49 #include "qplib_rcfw.h" 50 #include "qplib_sp.h" 51 #include "qplib_fp.h" 52 53 static void bnxt_qplib_service_creq(unsigned long data); 54 55 /* Hardware communication channel */ 56 static int __wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) 57 { 58 u16 cbit; 59 int rc; 60 61 cbit = cookie % rcfw->cmdq_depth; 62 rc = wait_event_timeout(rcfw->waitq, 63 !test_bit(cbit, rcfw->cmdq_bitmap), 64 msecs_to_jiffies(RCFW_CMD_WAIT_TIME_MS)); 65 return rc ? 0 : -ETIMEDOUT; 66 }; 67 68 static int __block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) 69 { 70 u32 count = RCFW_BLOCKED_CMD_WAIT_COUNT; 71 u16 cbit; 72 73 cbit = cookie % rcfw->cmdq_depth; 74 if (!test_bit(cbit, rcfw->cmdq_bitmap)) 75 goto done; 76 do { 77 mdelay(1); /* 1m sec */ 78 bnxt_qplib_service_creq((unsigned long)rcfw); 79 } while (test_bit(cbit, rcfw->cmdq_bitmap) && --count); 80 done: 81 return count ? 0 : -ETIMEDOUT; 82 }; 83 84 static int __send_message(struct bnxt_qplib_rcfw *rcfw, struct cmdq_base *req, 85 struct creq_base *resp, void *sb, u8 is_block) 86 { 87 struct bnxt_qplib_cmdqe *cmdqe, **cmdq_ptr; 88 struct bnxt_qplib_hwq *cmdq = &rcfw->cmdq; 89 u32 cmdq_depth = rcfw->cmdq_depth; 90 struct bnxt_qplib_crsq *crsqe; 91 u32 sw_prod, cmdq_prod; 92 unsigned long flags; 93 u32 size, opcode; 94 u16 cookie, cbit; 95 u8 *preq; 96 97 opcode = req->opcode; 98 if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags) && 99 (opcode != CMDQ_BASE_OPCODE_QUERY_FUNC && 100 opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW && 101 opcode != CMDQ_BASE_OPCODE_QUERY_VERSION)) { 102 dev_err(&rcfw->pdev->dev, 103 "RCFW not initialized, reject opcode 0x%x\n", opcode); 104 return -EINVAL; 105 } 106 107 if (test_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags) && 108 opcode == CMDQ_BASE_OPCODE_INITIALIZE_FW) { 109 dev_err(&rcfw->pdev->dev, "RCFW already initialized!\n"); 110 return -EINVAL; 111 } 112 113 if (test_bit(FIRMWARE_TIMED_OUT, &rcfw->flags)) 114 return -ETIMEDOUT; 115 116 /* Cmdq are in 16-byte units, each request can consume 1 or more 117 * cmdqe 118 */ 119 spin_lock_irqsave(&cmdq->lock, flags); 120 if (req->cmd_size >= HWQ_FREE_SLOTS(cmdq)) { 121 dev_err(&rcfw->pdev->dev, "RCFW: CMDQ is full!\n"); 122 spin_unlock_irqrestore(&cmdq->lock, flags); 123 return -EAGAIN; 124 } 125 126 127 cookie = rcfw->seq_num & RCFW_MAX_COOKIE_VALUE; 128 cbit = cookie % rcfw->cmdq_depth; 129 if (is_block) 130 cookie |= RCFW_CMD_IS_BLOCKING; 131 132 set_bit(cbit, rcfw->cmdq_bitmap); 133 req->cookie = cpu_to_le16(cookie); 134 crsqe = &rcfw->crsqe_tbl[cbit]; 135 if (crsqe->resp) { 136 spin_unlock_irqrestore(&cmdq->lock, flags); 137 return -EBUSY; 138 } 139 140 size = req->cmd_size; 141 /* change the cmd_size to the number of 16byte cmdq unit. 142 * req->cmd_size is modified here 143 */ 144 bnxt_qplib_set_cmd_slots(req); 145 146 memset(resp, 0, sizeof(*resp)); 147 crsqe->resp = (struct creq_qp_event *)resp; 148 crsqe->resp->cookie = req->cookie; 149 crsqe->req_size = req->cmd_size; 150 if (req->resp_size && sb) { 151 struct bnxt_qplib_rcfw_sbuf *sbuf = sb; 152 153 req->resp_addr = cpu_to_le64(sbuf->dma_addr); 154 req->resp_size = (sbuf->size + BNXT_QPLIB_CMDQE_UNITS - 1) / 155 BNXT_QPLIB_CMDQE_UNITS; 156 } 157 158 cmdq_ptr = (struct bnxt_qplib_cmdqe **)cmdq->pbl_ptr; 159 preq = (u8 *)req; 160 do { 161 /* Locate the next cmdq slot */ 162 sw_prod = HWQ_CMP(cmdq->prod, cmdq); 163 cmdqe = &cmdq_ptr[get_cmdq_pg(sw_prod, cmdq_depth)] 164 [get_cmdq_idx(sw_prod, cmdq_depth)]; 165 if (!cmdqe) { 166 dev_err(&rcfw->pdev->dev, 167 "RCFW request failed with no cmdqe!\n"); 168 goto done; 169 } 170 /* Copy a segment of the req cmd to the cmdq */ 171 memset(cmdqe, 0, sizeof(*cmdqe)); 172 memcpy(cmdqe, preq, min_t(u32, size, sizeof(*cmdqe))); 173 preq += min_t(u32, size, sizeof(*cmdqe)); 174 size -= min_t(u32, size, sizeof(*cmdqe)); 175 cmdq->prod++; 176 rcfw->seq_num++; 177 } while (size > 0); 178 179 rcfw->seq_num++; 180 181 cmdq_prod = cmdq->prod; 182 if (test_bit(FIRMWARE_FIRST_FLAG, &rcfw->flags)) { 183 /* The very first doorbell write 184 * is required to set this flag 185 * which prompts the FW to reset 186 * its internal pointers 187 */ 188 cmdq_prod |= BIT(FIRMWARE_FIRST_FLAG); 189 clear_bit(FIRMWARE_FIRST_FLAG, &rcfw->flags); 190 } 191 192 /* ring CMDQ DB */ 193 wmb(); 194 writel(cmdq_prod, rcfw->cmdq_bar_reg_iomem + 195 rcfw->cmdq_bar_reg_prod_off); 196 writel(RCFW_CMDQ_TRIG_VAL, rcfw->cmdq_bar_reg_iomem + 197 rcfw->cmdq_bar_reg_trig_off); 198 done: 199 spin_unlock_irqrestore(&cmdq->lock, flags); 200 /* Return the CREQ response pointer */ 201 return 0; 202 } 203 204 int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, 205 struct cmdq_base *req, 206 struct creq_base *resp, 207 void *sb, u8 is_block) 208 { 209 struct creq_qp_event *evnt = (struct creq_qp_event *)resp; 210 u16 cookie; 211 u8 opcode, retry_cnt = 0xFF; 212 int rc = 0; 213 214 do { 215 opcode = req->opcode; 216 rc = __send_message(rcfw, req, resp, sb, is_block); 217 cookie = le16_to_cpu(req->cookie) & RCFW_MAX_COOKIE_VALUE; 218 if (!rc) 219 break; 220 221 if (!retry_cnt || (rc != -EAGAIN && rc != -EBUSY)) { 222 /* send failed */ 223 dev_err(&rcfw->pdev->dev, "cmdq[%#x]=%#x send failed\n", 224 cookie, opcode); 225 return rc; 226 } 227 is_block ? mdelay(1) : usleep_range(500, 1000); 228 229 } while (retry_cnt--); 230 231 if (is_block) 232 rc = __block_for_resp(rcfw, cookie); 233 else 234 rc = __wait_for_resp(rcfw, cookie); 235 if (rc) { 236 /* timed out */ 237 dev_err(&rcfw->pdev->dev, "cmdq[%#x]=%#x timedout (%d)msec\n", 238 cookie, opcode, RCFW_CMD_WAIT_TIME_MS); 239 set_bit(FIRMWARE_TIMED_OUT, &rcfw->flags); 240 return rc; 241 } 242 243 if (evnt->status) { 244 /* failed with status */ 245 dev_err(&rcfw->pdev->dev, "cmdq[%#x]=%#x status %#x\n", 246 cookie, opcode, evnt->status); 247 rc = -EFAULT; 248 } 249 250 return rc; 251 } 252 /* Completions */ 253 static int bnxt_qplib_process_func_event(struct bnxt_qplib_rcfw *rcfw, 254 struct creq_func_event *func_event) 255 { 256 switch (func_event->event) { 257 case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR: 258 break; 259 case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR: 260 break; 261 case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR: 262 break; 263 case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR: 264 break; 265 case CREQ_FUNC_EVENT_EVENT_CQ_ERROR: 266 break; 267 case CREQ_FUNC_EVENT_EVENT_TQM_ERROR: 268 break; 269 case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR: 270 break; 271 case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR: 272 /* SRQ ctx error, call srq_handler?? 273 * But there's no SRQ handle! 274 */ 275 break; 276 case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR: 277 break; 278 case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR: 279 break; 280 case CREQ_FUNC_EVENT_EVENT_TIM_ERROR: 281 break; 282 case CREQ_FUNC_EVENT_EVENT_VF_COMM_REQUEST: 283 break; 284 case CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED: 285 break; 286 default: 287 return -EINVAL; 288 } 289 return 0; 290 } 291 292 static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw, 293 struct creq_qp_event *qp_event) 294 { 295 struct bnxt_qplib_hwq *cmdq = &rcfw->cmdq; 296 struct creq_qp_error_notification *err_event; 297 struct bnxt_qplib_crsq *crsqe; 298 unsigned long flags; 299 struct bnxt_qplib_qp *qp; 300 u16 cbit, blocked = 0; 301 u16 cookie; 302 __le16 mcookie; 303 u32 qp_id; 304 305 switch (qp_event->event) { 306 case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION: 307 err_event = (struct creq_qp_error_notification *)qp_event; 308 qp_id = le32_to_cpu(err_event->xid); 309 qp = rcfw->qp_tbl[qp_id].qp_handle; 310 dev_dbg(&rcfw->pdev->dev, 311 "Received QP error notification\n"); 312 dev_dbg(&rcfw->pdev->dev, 313 "qpid 0x%x, req_err=0x%x, resp_err=0x%x\n", 314 qp_id, err_event->req_err_state_reason, 315 err_event->res_err_state_reason); 316 if (!qp) 317 break; 318 bnxt_qplib_mark_qp_error(qp); 319 rcfw->aeq_handler(rcfw, qp_event, qp); 320 break; 321 default: 322 /* 323 * Command Response 324 * cmdq->lock needs to be acquired to synchronie 325 * the command send and completion reaping. This function 326 * is always called with creq->lock held. Using 327 * the nested variant of spin_lock. 328 * 329 */ 330 331 spin_lock_irqsave_nested(&cmdq->lock, flags, 332 SINGLE_DEPTH_NESTING); 333 cookie = le16_to_cpu(qp_event->cookie); 334 mcookie = qp_event->cookie; 335 blocked = cookie & RCFW_CMD_IS_BLOCKING; 336 cookie &= RCFW_MAX_COOKIE_VALUE; 337 cbit = cookie % rcfw->cmdq_depth; 338 crsqe = &rcfw->crsqe_tbl[cbit]; 339 if (crsqe->resp && 340 crsqe->resp->cookie == mcookie) { 341 memcpy(crsqe->resp, qp_event, sizeof(*qp_event)); 342 crsqe->resp = NULL; 343 } else { 344 if (crsqe->resp && crsqe->resp->cookie) 345 dev_err(&rcfw->pdev->dev, 346 "CMD %s cookie sent=%#x, recd=%#x\n", 347 crsqe->resp ? "mismatch" : "collision", 348 crsqe->resp ? crsqe->resp->cookie : 0, 349 mcookie); 350 } 351 if (!test_and_clear_bit(cbit, rcfw->cmdq_bitmap)) 352 dev_warn(&rcfw->pdev->dev, 353 "CMD bit %d was not requested\n", cbit); 354 cmdq->cons += crsqe->req_size; 355 crsqe->req_size = 0; 356 357 if (!blocked) 358 wake_up(&rcfw->waitq); 359 spin_unlock_irqrestore(&cmdq->lock, flags); 360 } 361 return 0; 362 } 363 364 /* SP - CREQ Completion handlers */ 365 static void bnxt_qplib_service_creq(unsigned long data) 366 { 367 struct bnxt_qplib_rcfw *rcfw = (struct bnxt_qplib_rcfw *)data; 368 bool gen_p5 = bnxt_qplib_is_chip_gen_p5(rcfw->res->cctx); 369 struct bnxt_qplib_hwq *creq = &rcfw->creq; 370 u32 type, budget = CREQ_ENTRY_POLL_BUDGET; 371 struct creq_base *creqe, **creq_ptr; 372 u32 sw_cons, raw_cons; 373 unsigned long flags; 374 375 /* Service the CREQ until budget is over */ 376 spin_lock_irqsave(&creq->lock, flags); 377 raw_cons = creq->cons; 378 while (budget > 0) { 379 sw_cons = HWQ_CMP(raw_cons, creq); 380 creq_ptr = (struct creq_base **)creq->pbl_ptr; 381 creqe = &creq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)]; 382 if (!CREQ_CMP_VALID(creqe, raw_cons, creq->max_elements)) 383 break; 384 /* The valid test of the entry must be done first before 385 * reading any further. 386 */ 387 dma_rmb(); 388 389 type = creqe->type & CREQ_BASE_TYPE_MASK; 390 switch (type) { 391 case CREQ_BASE_TYPE_QP_EVENT: 392 bnxt_qplib_process_qp_event 393 (rcfw, (struct creq_qp_event *)creqe); 394 rcfw->creq_qp_event_processed++; 395 break; 396 case CREQ_BASE_TYPE_FUNC_EVENT: 397 if (!bnxt_qplib_process_func_event 398 (rcfw, (struct creq_func_event *)creqe)) 399 rcfw->creq_func_event_processed++; 400 else 401 dev_warn(&rcfw->pdev->dev, 402 "aeqe:%#x Not handled\n", type); 403 break; 404 default: 405 if (type != ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT) 406 dev_warn(&rcfw->pdev->dev, 407 "creqe with event 0x%x not handled\n", 408 type); 409 break; 410 } 411 raw_cons++; 412 budget--; 413 } 414 415 if (creq->cons != raw_cons) { 416 creq->cons = raw_cons; 417 bnxt_qplib_ring_creq_db_rearm(rcfw->creq_bar_reg_iomem, 418 raw_cons, creq->max_elements, 419 rcfw->creq_ring_id, gen_p5); 420 } 421 spin_unlock_irqrestore(&creq->lock, flags); 422 } 423 424 static irqreturn_t bnxt_qplib_creq_irq(int irq, void *dev_instance) 425 { 426 struct bnxt_qplib_rcfw *rcfw = dev_instance; 427 struct bnxt_qplib_hwq *creq = &rcfw->creq; 428 struct creq_base **creq_ptr; 429 u32 sw_cons; 430 431 /* Prefetch the CREQ element */ 432 sw_cons = HWQ_CMP(creq->cons, creq); 433 creq_ptr = (struct creq_base **)rcfw->creq.pbl_ptr; 434 prefetch(&creq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)]); 435 436 tasklet_schedule(&rcfw->worker); 437 438 return IRQ_HANDLED; 439 } 440 441 /* RCFW */ 442 int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw) 443 { 444 struct cmdq_deinitialize_fw req; 445 struct creq_deinitialize_fw_resp resp; 446 u16 cmd_flags = 0; 447 int rc; 448 449 RCFW_CMD_PREP(req, DEINITIALIZE_FW, cmd_flags); 450 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, 451 NULL, 0); 452 if (rc) 453 return rc; 454 455 clear_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags); 456 return 0; 457 } 458 459 static int __get_pbl_pg_idx(struct bnxt_qplib_pbl *pbl) 460 { 461 return (pbl->pg_size == ROCE_PG_SIZE_4K ? 462 CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K : 463 pbl->pg_size == ROCE_PG_SIZE_8K ? 464 CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K : 465 pbl->pg_size == ROCE_PG_SIZE_64K ? 466 CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K : 467 pbl->pg_size == ROCE_PG_SIZE_2M ? 468 CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M : 469 pbl->pg_size == ROCE_PG_SIZE_8M ? 470 CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M : 471 pbl->pg_size == ROCE_PG_SIZE_1G ? 472 CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G : 473 CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K); 474 } 475 476 int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, 477 struct bnxt_qplib_ctx *ctx, int is_virtfn) 478 { 479 struct cmdq_initialize_fw req; 480 struct creq_initialize_fw_resp resp; 481 u16 cmd_flags = 0, level; 482 int rc; 483 484 RCFW_CMD_PREP(req, INITIALIZE_FW, cmd_flags); 485 /* Supply (log-base-2-of-host-page-size - base-page-shift) 486 * to bono to adjust the doorbell page sizes. 487 */ 488 req.log2_dbr_pg_size = cpu_to_le16(PAGE_SHIFT - 489 RCFW_DBR_BASE_PAGE_SHIFT); 490 /* 491 * Gen P5 devices doesn't require this allocation 492 * as the L2 driver does the same for RoCE also. 493 * Also, VFs need not setup the HW context area, PF 494 * shall setup this area for VF. Skipping the 495 * HW programming 496 */ 497 if (is_virtfn || bnxt_qplib_is_chip_gen_p5(rcfw->res->cctx)) 498 goto skip_ctx_setup; 499 500 level = ctx->qpc_tbl.level; 501 req.qpc_pg_size_qpc_lvl = (level << CMDQ_INITIALIZE_FW_QPC_LVL_SFT) | 502 __get_pbl_pg_idx(&ctx->qpc_tbl.pbl[level]); 503 level = ctx->mrw_tbl.level; 504 req.mrw_pg_size_mrw_lvl = (level << CMDQ_INITIALIZE_FW_MRW_LVL_SFT) | 505 __get_pbl_pg_idx(&ctx->mrw_tbl.pbl[level]); 506 level = ctx->srqc_tbl.level; 507 req.srq_pg_size_srq_lvl = (level << CMDQ_INITIALIZE_FW_SRQ_LVL_SFT) | 508 __get_pbl_pg_idx(&ctx->srqc_tbl.pbl[level]); 509 level = ctx->cq_tbl.level; 510 req.cq_pg_size_cq_lvl = (level << CMDQ_INITIALIZE_FW_CQ_LVL_SFT) | 511 __get_pbl_pg_idx(&ctx->cq_tbl.pbl[level]); 512 level = ctx->srqc_tbl.level; 513 req.srq_pg_size_srq_lvl = (level << CMDQ_INITIALIZE_FW_SRQ_LVL_SFT) | 514 __get_pbl_pg_idx(&ctx->srqc_tbl.pbl[level]); 515 level = ctx->cq_tbl.level; 516 req.cq_pg_size_cq_lvl = (level << CMDQ_INITIALIZE_FW_CQ_LVL_SFT) | 517 __get_pbl_pg_idx(&ctx->cq_tbl.pbl[level]); 518 level = ctx->tim_tbl.level; 519 req.tim_pg_size_tim_lvl = (level << CMDQ_INITIALIZE_FW_TIM_LVL_SFT) | 520 __get_pbl_pg_idx(&ctx->tim_tbl.pbl[level]); 521 level = ctx->tqm_pde_level; 522 req.tqm_pg_size_tqm_lvl = (level << CMDQ_INITIALIZE_FW_TQM_LVL_SFT) | 523 __get_pbl_pg_idx(&ctx->tqm_pde.pbl[level]); 524 525 req.qpc_page_dir = 526 cpu_to_le64(ctx->qpc_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); 527 req.mrw_page_dir = 528 cpu_to_le64(ctx->mrw_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); 529 req.srq_page_dir = 530 cpu_to_le64(ctx->srqc_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); 531 req.cq_page_dir = 532 cpu_to_le64(ctx->cq_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); 533 req.tim_page_dir = 534 cpu_to_le64(ctx->tim_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); 535 req.tqm_page_dir = 536 cpu_to_le64(ctx->tqm_pde.pbl[PBL_LVL_0].pg_map_arr[0]); 537 538 req.number_of_qp = cpu_to_le32(ctx->qpc_tbl.max_elements); 539 req.number_of_mrw = cpu_to_le32(ctx->mrw_tbl.max_elements); 540 req.number_of_srq = cpu_to_le32(ctx->srqc_tbl.max_elements); 541 req.number_of_cq = cpu_to_le32(ctx->cq_tbl.max_elements); 542 543 req.max_qp_per_vf = cpu_to_le32(ctx->vf_res.max_qp_per_vf); 544 req.max_mrw_per_vf = cpu_to_le32(ctx->vf_res.max_mrw_per_vf); 545 req.max_srq_per_vf = cpu_to_le32(ctx->vf_res.max_srq_per_vf); 546 req.max_cq_per_vf = cpu_to_le32(ctx->vf_res.max_cq_per_vf); 547 req.max_gid_per_vf = cpu_to_le32(ctx->vf_res.max_gid_per_vf); 548 549 skip_ctx_setup: 550 req.stat_ctx_id = cpu_to_le32(ctx->stats.fw_id); 551 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, 552 NULL, 0); 553 if (rc) 554 return rc; 555 set_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags); 556 return 0; 557 } 558 559 void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw) 560 { 561 kfree(rcfw->qp_tbl); 562 kfree(rcfw->crsqe_tbl); 563 bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->cmdq); 564 bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->creq); 565 rcfw->pdev = NULL; 566 } 567 568 int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev, 569 struct bnxt_qplib_rcfw *rcfw, 570 struct bnxt_qplib_ctx *ctx, 571 int qp_tbl_sz) 572 { 573 u8 hwq_type; 574 575 rcfw->pdev = pdev; 576 rcfw->creq.max_elements = BNXT_QPLIB_CREQE_MAX_CNT; 577 hwq_type = bnxt_qplib_get_hwq_type(rcfw->res); 578 if (bnxt_qplib_alloc_init_hwq(rcfw->pdev, &rcfw->creq, NULL, 579 &rcfw->creq.max_elements, 580 BNXT_QPLIB_CREQE_UNITS, 581 0, PAGE_SIZE, hwq_type)) { 582 dev_err(&rcfw->pdev->dev, 583 "HW channel CREQ allocation failed\n"); 584 goto fail; 585 } 586 if (ctx->hwrm_intf_ver < HWRM_VERSION_RCFW_CMDQ_DEPTH_CHECK) 587 rcfw->cmdq_depth = BNXT_QPLIB_CMDQE_MAX_CNT_256; 588 else 589 rcfw->cmdq_depth = BNXT_QPLIB_CMDQE_MAX_CNT_8192; 590 591 rcfw->cmdq.max_elements = rcfw->cmdq_depth; 592 if (bnxt_qplib_alloc_init_hwq 593 (rcfw->pdev, &rcfw->cmdq, NULL, 594 &rcfw->cmdq.max_elements, 595 BNXT_QPLIB_CMDQE_UNITS, 0, 596 bnxt_qplib_cmdqe_page_size(rcfw->cmdq_depth), 597 HWQ_TYPE_CTX)) { 598 dev_err(&rcfw->pdev->dev, 599 "HW channel CMDQ allocation failed\n"); 600 goto fail; 601 } 602 603 rcfw->crsqe_tbl = kcalloc(rcfw->cmdq.max_elements, 604 sizeof(*rcfw->crsqe_tbl), GFP_KERNEL); 605 if (!rcfw->crsqe_tbl) 606 goto fail; 607 608 rcfw->qp_tbl_size = qp_tbl_sz; 609 rcfw->qp_tbl = kcalloc(qp_tbl_sz, sizeof(struct bnxt_qplib_qp_node), 610 GFP_KERNEL); 611 if (!rcfw->qp_tbl) 612 goto fail; 613 614 return 0; 615 616 fail: 617 bnxt_qplib_free_rcfw_channel(rcfw); 618 return -ENOMEM; 619 } 620 621 void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill) 622 { 623 bool gen_p5 = bnxt_qplib_is_chip_gen_p5(rcfw->res->cctx); 624 625 tasklet_disable(&rcfw->worker); 626 /* Mask h/w interrupts */ 627 bnxt_qplib_ring_creq_db(rcfw->creq_bar_reg_iomem, rcfw->creq.cons, 628 rcfw->creq.max_elements, rcfw->creq_ring_id, 629 gen_p5); 630 /* Sync with last running IRQ-handler */ 631 synchronize_irq(rcfw->vector); 632 if (kill) 633 tasklet_kill(&rcfw->worker); 634 635 if (rcfw->requested) { 636 free_irq(rcfw->vector, rcfw); 637 rcfw->requested = false; 638 } 639 } 640 641 void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw) 642 { 643 unsigned long indx; 644 645 bnxt_qplib_rcfw_stop_irq(rcfw, true); 646 647 iounmap(rcfw->cmdq_bar_reg_iomem); 648 iounmap(rcfw->creq_bar_reg_iomem); 649 650 indx = find_first_bit(rcfw->cmdq_bitmap, rcfw->bmap_size); 651 if (indx != rcfw->bmap_size) 652 dev_err(&rcfw->pdev->dev, 653 "disabling RCFW with pending cmd-bit %lx\n", indx); 654 kfree(rcfw->cmdq_bitmap); 655 rcfw->bmap_size = 0; 656 657 rcfw->cmdq_bar_reg_iomem = NULL; 658 rcfw->creq_bar_reg_iomem = NULL; 659 rcfw->aeq_handler = NULL; 660 rcfw->vector = 0; 661 } 662 663 int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector, 664 bool need_init) 665 { 666 bool gen_p5 = bnxt_qplib_is_chip_gen_p5(rcfw->res->cctx); 667 int rc; 668 669 if (rcfw->requested) 670 return -EFAULT; 671 672 rcfw->vector = msix_vector; 673 if (need_init) 674 tasklet_init(&rcfw->worker, 675 bnxt_qplib_service_creq, (unsigned long)rcfw); 676 else 677 tasklet_enable(&rcfw->worker); 678 rc = request_irq(rcfw->vector, bnxt_qplib_creq_irq, 0, 679 "bnxt_qplib_creq", rcfw); 680 if (rc) 681 return rc; 682 rcfw->requested = true; 683 bnxt_qplib_ring_creq_db_rearm(rcfw->creq_bar_reg_iomem, 684 rcfw->creq.cons, rcfw->creq.max_elements, 685 rcfw->creq_ring_id, gen_p5); 686 687 return 0; 688 } 689 690 int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev, 691 struct bnxt_qplib_rcfw *rcfw, 692 int msix_vector, 693 int cp_bar_reg_off, int virt_fn, 694 int (*aeq_handler)(struct bnxt_qplib_rcfw *, 695 void *, void *)) 696 { 697 resource_size_t res_base; 698 struct cmdq_init init; 699 u16 bmap_size; 700 int rc; 701 702 /* General */ 703 rcfw->seq_num = 0; 704 set_bit(FIRMWARE_FIRST_FLAG, &rcfw->flags); 705 bmap_size = BITS_TO_LONGS(rcfw->cmdq_depth) * sizeof(unsigned long); 706 rcfw->cmdq_bitmap = kzalloc(bmap_size, GFP_KERNEL); 707 if (!rcfw->cmdq_bitmap) 708 return -ENOMEM; 709 rcfw->bmap_size = bmap_size; 710 711 /* CMDQ */ 712 rcfw->cmdq_bar_reg = RCFW_COMM_PCI_BAR_REGION; 713 res_base = pci_resource_start(pdev, rcfw->cmdq_bar_reg); 714 if (!res_base) 715 return -ENOMEM; 716 717 rcfw->cmdq_bar_reg_iomem = ioremap_nocache(res_base + 718 RCFW_COMM_BASE_OFFSET, 719 RCFW_COMM_SIZE); 720 if (!rcfw->cmdq_bar_reg_iomem) { 721 dev_err(&rcfw->pdev->dev, "CMDQ BAR region %d mapping failed\n", 722 rcfw->cmdq_bar_reg); 723 return -ENOMEM; 724 } 725 726 rcfw->cmdq_bar_reg_prod_off = virt_fn ? RCFW_VF_COMM_PROD_OFFSET : 727 RCFW_PF_COMM_PROD_OFFSET; 728 729 rcfw->cmdq_bar_reg_trig_off = RCFW_COMM_TRIG_OFFSET; 730 731 /* CREQ */ 732 rcfw->creq_bar_reg = RCFW_COMM_CONS_PCI_BAR_REGION; 733 res_base = pci_resource_start(pdev, rcfw->creq_bar_reg); 734 if (!res_base) 735 dev_err(&rcfw->pdev->dev, 736 "CREQ BAR region %d resc start is 0!\n", 737 rcfw->creq_bar_reg); 738 /* Unconditionally map 8 bytes to support 57500 series */ 739 rcfw->creq_bar_reg_iomem = ioremap_nocache(res_base + cp_bar_reg_off, 740 8); 741 if (!rcfw->creq_bar_reg_iomem) { 742 dev_err(&rcfw->pdev->dev, "CREQ BAR region %d mapping failed\n", 743 rcfw->creq_bar_reg); 744 iounmap(rcfw->cmdq_bar_reg_iomem); 745 rcfw->cmdq_bar_reg_iomem = NULL; 746 return -ENOMEM; 747 } 748 rcfw->creq_qp_event_processed = 0; 749 rcfw->creq_func_event_processed = 0; 750 751 if (aeq_handler) 752 rcfw->aeq_handler = aeq_handler; 753 init_waitqueue_head(&rcfw->waitq); 754 755 rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_vector, true); 756 if (rc) { 757 dev_err(&rcfw->pdev->dev, 758 "Failed to request IRQ for CREQ rc = 0x%x\n", rc); 759 bnxt_qplib_disable_rcfw_channel(rcfw); 760 return rc; 761 } 762 763 init.cmdq_pbl = cpu_to_le64(rcfw->cmdq.pbl[PBL_LVL_0].pg_map_arr[0]); 764 init.cmdq_size_cmdq_lvl = cpu_to_le16( 765 ((rcfw->cmdq_depth << CMDQ_INIT_CMDQ_SIZE_SFT) & 766 CMDQ_INIT_CMDQ_SIZE_MASK) | 767 ((rcfw->cmdq.level << CMDQ_INIT_CMDQ_LVL_SFT) & 768 CMDQ_INIT_CMDQ_LVL_MASK)); 769 init.creq_ring_id = cpu_to_le16(rcfw->creq_ring_id); 770 771 /* Write to the Bono mailbox register */ 772 __iowrite32_copy(rcfw->cmdq_bar_reg_iomem, &init, sizeof(init) / 4); 773 return 0; 774 } 775 776 struct bnxt_qplib_rcfw_sbuf *bnxt_qplib_rcfw_alloc_sbuf( 777 struct bnxt_qplib_rcfw *rcfw, 778 u32 size) 779 { 780 struct bnxt_qplib_rcfw_sbuf *sbuf; 781 782 sbuf = kzalloc(sizeof(*sbuf), GFP_ATOMIC); 783 if (!sbuf) 784 return NULL; 785 786 sbuf->size = size; 787 sbuf->sb = dma_alloc_coherent(&rcfw->pdev->dev, sbuf->size, 788 &sbuf->dma_addr, GFP_ATOMIC); 789 if (!sbuf->sb) 790 goto bail; 791 792 return sbuf; 793 bail: 794 kfree(sbuf); 795 return NULL; 796 } 797 798 void bnxt_qplib_rcfw_free_sbuf(struct bnxt_qplib_rcfw *rcfw, 799 struct bnxt_qplib_rcfw_sbuf *sbuf) 800 { 801 if (sbuf->sb) 802 dma_free_coherent(&rcfw->pdev->dev, sbuf->size, 803 sbuf->sb, sbuf->dma_addr); 804 kfree(sbuf); 805 } 806