1 /* QLogic qed NIC Driver 2 * Copyright (c) 2015-2017 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/types.h> 34 #include <asm/byteorder.h> 35 #include <linux/io.h> 36 #include <linux/delay.h> 37 #include <linux/dma-mapping.h> 38 #include <linux/errno.h> 39 #include <linux/kernel.h> 40 #include <linux/list.h> 41 #include <linux/pci.h> 42 #include <linux/slab.h> 43 #include <linux/spinlock.h> 44 #include <linux/string.h> 45 #include "qed.h" 46 #include "qed_cxt.h" 47 #include "qed_dev_api.h" 48 #include "qed_hsi.h" 49 #include "qed_hw.h" 50 #include "qed_int.h" 51 #include "qed_iscsi.h" 52 #include "qed_mcp.h" 53 #include "qed_ooo.h" 54 #include "qed_reg_addr.h" 55 #include "qed_sp.h" 56 #include "qed_sriov.h" 57 #include "qed_roce.h" 58 59 /*************************************************************************** 60 * Structures & Definitions 61 ***************************************************************************/ 62 63 #define SPQ_HIGH_PRI_RESERVE_DEFAULT (1) 64 65 #define SPQ_BLOCK_DELAY_MAX_ITER (10) 66 #define SPQ_BLOCK_DELAY_US (10) 67 #define SPQ_BLOCK_SLEEP_MAX_ITER (1000) 68 #define SPQ_BLOCK_SLEEP_MS (5) 69 70 /*************************************************************************** 71 * Blocking Imp. (BLOCK/EBLOCK mode) 72 ***************************************************************************/ 73 static void qed_spq_blocking_cb(struct qed_hwfn *p_hwfn, 74 void *cookie, 75 union event_ring_data *data, u8 fw_return_code) 76 { 77 struct qed_spq_comp_done *comp_done; 78 79 comp_done = (struct qed_spq_comp_done *)cookie; 80 81 comp_done->fw_return_code = fw_return_code; 82 83 /* Make sure completion done is visible on waiting thread */ 84 smp_store_release(&comp_done->done, 0x1); 85 } 86 87 static int __qed_spq_block(struct qed_hwfn *p_hwfn, 88 struct qed_spq_entry *p_ent, 89 u8 *p_fw_ret, bool sleep_between_iter) 90 { 91 struct qed_spq_comp_done *comp_done; 92 u32 iter_cnt; 93 94 comp_done = (struct qed_spq_comp_done *)p_ent->comp_cb.cookie; 95 iter_cnt = sleep_between_iter ? SPQ_BLOCK_SLEEP_MAX_ITER 96 : SPQ_BLOCK_DELAY_MAX_ITER; 97 98 while (iter_cnt--) { 99 /* Validate we receive completion update */ 100 if (READ_ONCE(comp_done->done) == 1) { 101 /* Read updated FW return value */ 102 smp_read_barrier_depends(); 103 if (p_fw_ret) 104 *p_fw_ret = comp_done->fw_return_code; 105 return 0; 106 } 107 108 if (sleep_between_iter) 109 msleep(SPQ_BLOCK_SLEEP_MS); 110 else 111 udelay(SPQ_BLOCK_DELAY_US); 112 } 113 114 return -EBUSY; 115 } 116 117 static int qed_spq_block(struct qed_hwfn *p_hwfn, 118 struct qed_spq_entry *p_ent, 119 u8 *p_fw_ret, bool skip_quick_poll) 120 { 121 struct qed_spq_comp_done *comp_done; 122 int rc; 123 124 /* A relatively short polling period w/o sleeping, to allow the FW to 125 * complete the ramrod and thus possibly to avoid the following sleeps. 126 */ 127 if (!skip_quick_poll) { 128 rc = __qed_spq_block(p_hwfn, p_ent, p_fw_ret, false); 129 if (!rc) 130 return 0; 131 } 132 133 /* Move to polling with a sleeping period between iterations */ 134 rc = __qed_spq_block(p_hwfn, p_ent, p_fw_ret, true); 135 if (!rc) 136 return 0; 137 138 DP_INFO(p_hwfn, "Ramrod is stuck, requesting MCP drain\n"); 139 rc = qed_mcp_drain(p_hwfn, p_hwfn->p_main_ptt); 140 if (rc) { 141 DP_NOTICE(p_hwfn, "MCP drain failed\n"); 142 goto err; 143 } 144 145 /* Retry after drain */ 146 rc = __qed_spq_block(p_hwfn, p_ent, p_fw_ret, true); 147 if (!rc) 148 return 0; 149 150 comp_done = (struct qed_spq_comp_done *)p_ent->comp_cb.cookie; 151 if (comp_done->done == 1) { 152 if (p_fw_ret) 153 *p_fw_ret = comp_done->fw_return_code; 154 return 0; 155 } 156 err: 157 DP_NOTICE(p_hwfn, 158 "Ramrod is stuck [CID %08x cmd %02x protocol %02x echo %04x]\n", 159 le32_to_cpu(p_ent->elem.hdr.cid), 160 p_ent->elem.hdr.cmd_id, 161 p_ent->elem.hdr.protocol_id, 162 le16_to_cpu(p_ent->elem.hdr.echo)); 163 164 return -EBUSY; 165 } 166 167 /*************************************************************************** 168 * SPQ entries inner API 169 ***************************************************************************/ 170 static int qed_spq_fill_entry(struct qed_hwfn *p_hwfn, 171 struct qed_spq_entry *p_ent) 172 { 173 p_ent->flags = 0; 174 175 switch (p_ent->comp_mode) { 176 case QED_SPQ_MODE_EBLOCK: 177 case QED_SPQ_MODE_BLOCK: 178 p_ent->comp_cb.function = qed_spq_blocking_cb; 179 break; 180 case QED_SPQ_MODE_CB: 181 break; 182 default: 183 DP_NOTICE(p_hwfn, "Unknown SPQE completion mode %d\n", 184 p_ent->comp_mode); 185 return -EINVAL; 186 } 187 188 DP_VERBOSE(p_hwfn, QED_MSG_SPQ, 189 "Ramrod header: [CID 0x%08x CMD 0x%02x protocol 0x%02x] Data pointer: [%08x:%08x] Completion Mode: %s\n", 190 p_ent->elem.hdr.cid, 191 p_ent->elem.hdr.cmd_id, 192 p_ent->elem.hdr.protocol_id, 193 p_ent->elem.data_ptr.hi, 194 p_ent->elem.data_ptr.lo, 195 D_TRINE(p_ent->comp_mode, QED_SPQ_MODE_EBLOCK, 196 QED_SPQ_MODE_BLOCK, "MODE_EBLOCK", "MODE_BLOCK", 197 "MODE_CB")); 198 199 return 0; 200 } 201 202 /*************************************************************************** 203 * HSI access 204 ***************************************************************************/ 205 static void qed_spq_hw_initialize(struct qed_hwfn *p_hwfn, 206 struct qed_spq *p_spq) 207 { 208 struct core_conn_context *p_cxt; 209 struct qed_cxt_info cxt_info; 210 u16 physical_q; 211 int rc; 212 213 cxt_info.iid = p_spq->cid; 214 215 rc = qed_cxt_get_cid_info(p_hwfn, &cxt_info); 216 217 if (rc < 0) { 218 DP_NOTICE(p_hwfn, "Cannot find context info for cid=%d\n", 219 p_spq->cid); 220 return; 221 } 222 223 p_cxt = cxt_info.p_cxt; 224 225 SET_FIELD(p_cxt->xstorm_ag_context.flags10, 226 XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN, 1); 227 SET_FIELD(p_cxt->xstorm_ag_context.flags1, 228 XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE, 1); 229 SET_FIELD(p_cxt->xstorm_ag_context.flags9, 230 XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN, 1); 231 232 /* QM physical queue */ 233 physical_q = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_LB); 234 p_cxt->xstorm_ag_context.physical_q0 = cpu_to_le16(physical_q); 235 236 p_cxt->xstorm_st_context.spq_base_lo = 237 DMA_LO_LE(p_spq->chain.p_phys_addr); 238 p_cxt->xstorm_st_context.spq_base_hi = 239 DMA_HI_LE(p_spq->chain.p_phys_addr); 240 241 DMA_REGPAIR_LE(p_cxt->xstorm_st_context.consolid_base_addr, 242 p_hwfn->p_consq->chain.p_phys_addr); 243 } 244 245 static int qed_spq_hw_post(struct qed_hwfn *p_hwfn, 246 struct qed_spq *p_spq, struct qed_spq_entry *p_ent) 247 { 248 struct qed_chain *p_chain = &p_hwfn->p_spq->chain; 249 u16 echo = qed_chain_get_prod_idx(p_chain); 250 struct slow_path_element *elem; 251 struct core_db_data db; 252 253 p_ent->elem.hdr.echo = cpu_to_le16(echo); 254 elem = qed_chain_produce(p_chain); 255 if (!elem) { 256 DP_NOTICE(p_hwfn, "Failed to produce from SPQ chain\n"); 257 return -EINVAL; 258 } 259 260 *elem = p_ent->elem; /* struct assignment */ 261 262 /* send a doorbell on the slow hwfn session */ 263 memset(&db, 0, sizeof(db)); 264 SET_FIELD(db.params, CORE_DB_DATA_DEST, DB_DEST_XCM); 265 SET_FIELD(db.params, CORE_DB_DATA_AGG_CMD, DB_AGG_CMD_SET); 266 SET_FIELD(db.params, CORE_DB_DATA_AGG_VAL_SEL, 267 DQ_XCM_CORE_SPQ_PROD_CMD); 268 db.agg_flags = DQ_XCM_CORE_DQ_CF_CMD; 269 db.spq_prod = cpu_to_le16(qed_chain_get_prod_idx(p_chain)); 270 271 /* make sure the SPQE is updated before the doorbell */ 272 wmb(); 273 274 DOORBELL(p_hwfn, qed_db_addr(p_spq->cid, DQ_DEMS_LEGACY), *(u32 *)&db); 275 276 /* make sure doorbell is rang */ 277 wmb(); 278 279 DP_VERBOSE(p_hwfn, QED_MSG_SPQ, 280 "Doorbelled [0x%08x, CID 0x%08x] with Flags: %02x agg_params: %02x, prod: %04x\n", 281 qed_db_addr(p_spq->cid, DQ_DEMS_LEGACY), 282 p_spq->cid, db.params, db.agg_flags, 283 qed_chain_get_prod_idx(p_chain)); 284 285 return 0; 286 } 287 288 /*************************************************************************** 289 * Asynchronous events 290 ***************************************************************************/ 291 static int 292 qed_async_event_completion(struct qed_hwfn *p_hwfn, 293 struct event_ring_entry *p_eqe) 294 { 295 switch (p_eqe->protocol_id) { 296 #if IS_ENABLED(CONFIG_QED_RDMA) 297 case PROTOCOLID_ROCE: 298 qed_roce_async_event(p_hwfn, p_eqe->opcode, 299 &p_eqe->data.rdma_data); 300 return 0; 301 #endif 302 case PROTOCOLID_COMMON: 303 return qed_sriov_eqe_event(p_hwfn, 304 p_eqe->opcode, 305 p_eqe->echo, &p_eqe->data); 306 case PROTOCOLID_ISCSI: 307 if (!IS_ENABLED(CONFIG_QED_ISCSI)) 308 return -EINVAL; 309 310 if (p_hwfn->p_iscsi_info->event_cb) { 311 struct qed_iscsi_info *p_iscsi = p_hwfn->p_iscsi_info; 312 313 return p_iscsi->event_cb(p_iscsi->event_context, 314 p_eqe->opcode, &p_eqe->data); 315 } else { 316 DP_NOTICE(p_hwfn, 317 "iSCSI async completion is not set\n"); 318 return -EINVAL; 319 } 320 default: 321 DP_NOTICE(p_hwfn, 322 "Unknown Async completion for protocol: %d\n", 323 p_eqe->protocol_id); 324 return -EINVAL; 325 } 326 } 327 328 /*************************************************************************** 329 * EQ API 330 ***************************************************************************/ 331 void qed_eq_prod_update(struct qed_hwfn *p_hwfn, u16 prod) 332 { 333 u32 addr = GTT_BAR0_MAP_REG_USDM_RAM + 334 USTORM_EQE_CONS_OFFSET(p_hwfn->rel_pf_id); 335 336 REG_WR16(p_hwfn, addr, prod); 337 338 /* keep prod updates ordered */ 339 mmiowb(); 340 } 341 342 int qed_eq_completion(struct qed_hwfn *p_hwfn, void *cookie) 343 { 344 struct qed_eq *p_eq = cookie; 345 struct qed_chain *p_chain = &p_eq->chain; 346 int rc = 0; 347 348 /* take a snapshot of the FW consumer */ 349 u16 fw_cons_idx = le16_to_cpu(*p_eq->p_fw_cons); 350 351 DP_VERBOSE(p_hwfn, QED_MSG_SPQ, "fw_cons_idx %x\n", fw_cons_idx); 352 353 /* Need to guarantee the fw_cons index we use points to a usuable 354 * element (to comply with our chain), so our macros would comply 355 */ 356 if ((fw_cons_idx & qed_chain_get_usable_per_page(p_chain)) == 357 qed_chain_get_usable_per_page(p_chain)) 358 fw_cons_idx += qed_chain_get_unusable_per_page(p_chain); 359 360 /* Complete current segment of eq entries */ 361 while (fw_cons_idx != qed_chain_get_cons_idx(p_chain)) { 362 struct event_ring_entry *p_eqe = qed_chain_consume(p_chain); 363 364 if (!p_eqe) { 365 rc = -EINVAL; 366 break; 367 } 368 369 DP_VERBOSE(p_hwfn, QED_MSG_SPQ, 370 "op %x prot %x res0 %x echo %x fwret %x flags %x\n", 371 p_eqe->opcode, 372 p_eqe->protocol_id, 373 p_eqe->reserved0, 374 le16_to_cpu(p_eqe->echo), 375 p_eqe->fw_return_code, 376 p_eqe->flags); 377 378 if (GET_FIELD(p_eqe->flags, EVENT_RING_ENTRY_ASYNC)) { 379 if (qed_async_event_completion(p_hwfn, p_eqe)) 380 rc = -EINVAL; 381 } else if (qed_spq_completion(p_hwfn, 382 p_eqe->echo, 383 p_eqe->fw_return_code, 384 &p_eqe->data)) { 385 rc = -EINVAL; 386 } 387 388 qed_chain_recycle_consumed(p_chain); 389 } 390 391 qed_eq_prod_update(p_hwfn, qed_chain_get_prod_idx(p_chain)); 392 393 return rc; 394 } 395 396 struct qed_eq *qed_eq_alloc(struct qed_hwfn *p_hwfn, u16 num_elem) 397 { 398 struct qed_eq *p_eq; 399 400 /* Allocate EQ struct */ 401 p_eq = kzalloc(sizeof(*p_eq), GFP_KERNEL); 402 if (!p_eq) 403 return NULL; 404 405 /* Allocate and initialize EQ chain*/ 406 if (qed_chain_alloc(p_hwfn->cdev, 407 QED_CHAIN_USE_TO_PRODUCE, 408 QED_CHAIN_MODE_PBL, 409 QED_CHAIN_CNT_TYPE_U16, 410 num_elem, 411 sizeof(union event_ring_element), 412 &p_eq->chain)) 413 goto eq_allocate_fail; 414 415 /* register EQ completion on the SP SB */ 416 qed_int_register_cb(p_hwfn, qed_eq_completion, 417 p_eq, &p_eq->eq_sb_index, &p_eq->p_fw_cons); 418 419 return p_eq; 420 421 eq_allocate_fail: 422 qed_eq_free(p_hwfn, p_eq); 423 return NULL; 424 } 425 426 void qed_eq_setup(struct qed_hwfn *p_hwfn, struct qed_eq *p_eq) 427 { 428 qed_chain_reset(&p_eq->chain); 429 } 430 431 void qed_eq_free(struct qed_hwfn *p_hwfn, struct qed_eq *p_eq) 432 { 433 if (!p_eq) 434 return; 435 qed_chain_free(p_hwfn->cdev, &p_eq->chain); 436 kfree(p_eq); 437 } 438 439 /*************************************************************************** 440 * CQE API - manipulate EQ functionality 441 ***************************************************************************/ 442 static int qed_cqe_completion(struct qed_hwfn *p_hwfn, 443 struct eth_slow_path_rx_cqe *cqe, 444 enum protocol_type protocol) 445 { 446 if (IS_VF(p_hwfn->cdev)) 447 return 0; 448 449 /* @@@tmp - it's possible we'll eventually want to handle some 450 * actual commands that can arrive here, but for now this is only 451 * used to complete the ramrod using the echo value on the cqe 452 */ 453 return qed_spq_completion(p_hwfn, cqe->echo, 0, NULL); 454 } 455 456 int qed_eth_cqe_completion(struct qed_hwfn *p_hwfn, 457 struct eth_slow_path_rx_cqe *cqe) 458 { 459 int rc; 460 461 rc = qed_cqe_completion(p_hwfn, cqe, PROTOCOLID_ETH); 462 if (rc) 463 DP_NOTICE(p_hwfn, 464 "Failed to handle RXQ CQE [cmd 0x%02x]\n", 465 cqe->ramrod_cmd_id); 466 467 return rc; 468 } 469 470 /*************************************************************************** 471 * Slow hwfn Queue (spq) 472 ***************************************************************************/ 473 void qed_spq_setup(struct qed_hwfn *p_hwfn) 474 { 475 struct qed_spq *p_spq = p_hwfn->p_spq; 476 struct qed_spq_entry *p_virt = NULL; 477 dma_addr_t p_phys = 0; 478 u32 i, capacity; 479 480 INIT_LIST_HEAD(&p_spq->pending); 481 INIT_LIST_HEAD(&p_spq->completion_pending); 482 INIT_LIST_HEAD(&p_spq->free_pool); 483 INIT_LIST_HEAD(&p_spq->unlimited_pending); 484 spin_lock_init(&p_spq->lock); 485 486 /* SPQ empty pool */ 487 p_phys = p_spq->p_phys + offsetof(struct qed_spq_entry, ramrod); 488 p_virt = p_spq->p_virt; 489 490 capacity = qed_chain_get_capacity(&p_spq->chain); 491 for (i = 0; i < capacity; i++) { 492 DMA_REGPAIR_LE(p_virt->elem.data_ptr, p_phys); 493 494 list_add_tail(&p_virt->list, &p_spq->free_pool); 495 496 p_virt++; 497 p_phys += sizeof(struct qed_spq_entry); 498 } 499 500 /* Statistics */ 501 p_spq->normal_count = 0; 502 p_spq->comp_count = 0; 503 p_spq->comp_sent_count = 0; 504 p_spq->unlimited_pending_count = 0; 505 506 bitmap_zero(p_spq->p_comp_bitmap, SPQ_RING_SIZE); 507 p_spq->comp_bitmap_idx = 0; 508 509 /* SPQ cid, cannot fail */ 510 qed_cxt_acquire_cid(p_hwfn, PROTOCOLID_CORE, &p_spq->cid); 511 qed_spq_hw_initialize(p_hwfn, p_spq); 512 513 /* reset the chain itself */ 514 qed_chain_reset(&p_spq->chain); 515 } 516 517 int qed_spq_alloc(struct qed_hwfn *p_hwfn) 518 { 519 struct qed_spq_entry *p_virt = NULL; 520 struct qed_spq *p_spq = NULL; 521 dma_addr_t p_phys = 0; 522 u32 capacity; 523 524 /* SPQ struct */ 525 p_spq = kzalloc(sizeof(struct qed_spq), GFP_KERNEL); 526 if (!p_spq) 527 return -ENOMEM; 528 529 /* SPQ ring */ 530 if (qed_chain_alloc(p_hwfn->cdev, 531 QED_CHAIN_USE_TO_PRODUCE, 532 QED_CHAIN_MODE_SINGLE, 533 QED_CHAIN_CNT_TYPE_U16, 534 0, /* N/A when the mode is SINGLE */ 535 sizeof(struct slow_path_element), 536 &p_spq->chain)) 537 goto spq_allocate_fail; 538 539 /* allocate and fill the SPQ elements (incl. ramrod data list) */ 540 capacity = qed_chain_get_capacity(&p_spq->chain); 541 p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 542 capacity * sizeof(struct qed_spq_entry), 543 &p_phys, GFP_KERNEL); 544 if (!p_virt) 545 goto spq_allocate_fail; 546 547 p_spq->p_virt = p_virt; 548 p_spq->p_phys = p_phys; 549 p_hwfn->p_spq = p_spq; 550 551 return 0; 552 553 spq_allocate_fail: 554 qed_chain_free(p_hwfn->cdev, &p_spq->chain); 555 kfree(p_spq); 556 return -ENOMEM; 557 } 558 559 void qed_spq_free(struct qed_hwfn *p_hwfn) 560 { 561 struct qed_spq *p_spq = p_hwfn->p_spq; 562 u32 capacity; 563 564 if (!p_spq) 565 return; 566 567 if (p_spq->p_virt) { 568 capacity = qed_chain_get_capacity(&p_spq->chain); 569 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 570 capacity * 571 sizeof(struct qed_spq_entry), 572 p_spq->p_virt, p_spq->p_phys); 573 } 574 575 qed_chain_free(p_hwfn->cdev, &p_spq->chain); 576 ; 577 kfree(p_spq); 578 } 579 580 int qed_spq_get_entry(struct qed_hwfn *p_hwfn, struct qed_spq_entry **pp_ent) 581 { 582 struct qed_spq *p_spq = p_hwfn->p_spq; 583 struct qed_spq_entry *p_ent = NULL; 584 int rc = 0; 585 586 spin_lock_bh(&p_spq->lock); 587 588 if (list_empty(&p_spq->free_pool)) { 589 p_ent = kzalloc(sizeof(*p_ent), GFP_ATOMIC); 590 if (!p_ent) { 591 DP_NOTICE(p_hwfn, 592 "Failed to allocate an SPQ entry for a pending ramrod\n"); 593 rc = -ENOMEM; 594 goto out_unlock; 595 } 596 p_ent->queue = &p_spq->unlimited_pending; 597 } else { 598 p_ent = list_first_entry(&p_spq->free_pool, 599 struct qed_spq_entry, list); 600 list_del(&p_ent->list); 601 p_ent->queue = &p_spq->pending; 602 } 603 604 *pp_ent = p_ent; 605 606 out_unlock: 607 spin_unlock_bh(&p_spq->lock); 608 return rc; 609 } 610 611 /* Locked variant; Should be called while the SPQ lock is taken */ 612 static void __qed_spq_return_entry(struct qed_hwfn *p_hwfn, 613 struct qed_spq_entry *p_ent) 614 { 615 list_add_tail(&p_ent->list, &p_hwfn->p_spq->free_pool); 616 } 617 618 void qed_spq_return_entry(struct qed_hwfn *p_hwfn, struct qed_spq_entry *p_ent) 619 { 620 spin_lock_bh(&p_hwfn->p_spq->lock); 621 __qed_spq_return_entry(p_hwfn, p_ent); 622 spin_unlock_bh(&p_hwfn->p_spq->lock); 623 } 624 625 /** 626 * @brief qed_spq_add_entry - adds a new entry to the pending 627 * list. Should be used while lock is being held. 628 * 629 * Addes an entry to the pending list is there is room (en empty 630 * element is available in the free_pool), or else places the 631 * entry in the unlimited_pending pool. 632 * 633 * @param p_hwfn 634 * @param p_ent 635 * @param priority 636 * 637 * @return int 638 */ 639 static int qed_spq_add_entry(struct qed_hwfn *p_hwfn, 640 struct qed_spq_entry *p_ent, 641 enum spq_priority priority) 642 { 643 struct qed_spq *p_spq = p_hwfn->p_spq; 644 645 if (p_ent->queue == &p_spq->unlimited_pending) { 646 647 if (list_empty(&p_spq->free_pool)) { 648 list_add_tail(&p_ent->list, &p_spq->unlimited_pending); 649 p_spq->unlimited_pending_count++; 650 651 return 0; 652 } else { 653 struct qed_spq_entry *p_en2; 654 655 p_en2 = list_first_entry(&p_spq->free_pool, 656 struct qed_spq_entry, list); 657 list_del(&p_en2->list); 658 659 /* Copy the ring element physical pointer to the new 660 * entry, since we are about to override the entire ring 661 * entry and don't want to lose the pointer. 662 */ 663 p_ent->elem.data_ptr = p_en2->elem.data_ptr; 664 665 *p_en2 = *p_ent; 666 667 /* EBLOCK responsible to free the allocated p_ent */ 668 if (p_ent->comp_mode != QED_SPQ_MODE_EBLOCK) 669 kfree(p_ent); 670 671 p_ent = p_en2; 672 } 673 } 674 675 /* entry is to be placed in 'pending' queue */ 676 switch (priority) { 677 case QED_SPQ_PRIORITY_NORMAL: 678 list_add_tail(&p_ent->list, &p_spq->pending); 679 p_spq->normal_count++; 680 break; 681 case QED_SPQ_PRIORITY_HIGH: 682 list_add(&p_ent->list, &p_spq->pending); 683 p_spq->high_count++; 684 break; 685 default: 686 return -EINVAL; 687 } 688 689 return 0; 690 } 691 692 /*************************************************************************** 693 * Accessor 694 ***************************************************************************/ 695 u32 qed_spq_get_cid(struct qed_hwfn *p_hwfn) 696 { 697 if (!p_hwfn->p_spq) 698 return 0xffffffff; /* illegal */ 699 return p_hwfn->p_spq->cid; 700 } 701 702 /*************************************************************************** 703 * Posting new Ramrods 704 ***************************************************************************/ 705 static int qed_spq_post_list(struct qed_hwfn *p_hwfn, 706 struct list_head *head, u32 keep_reserve) 707 { 708 struct qed_spq *p_spq = p_hwfn->p_spq; 709 int rc; 710 711 while (qed_chain_get_elem_left(&p_spq->chain) > keep_reserve && 712 !list_empty(head)) { 713 struct qed_spq_entry *p_ent = 714 list_first_entry(head, struct qed_spq_entry, list); 715 list_del(&p_ent->list); 716 list_add_tail(&p_ent->list, &p_spq->completion_pending); 717 p_spq->comp_sent_count++; 718 719 rc = qed_spq_hw_post(p_hwfn, p_spq, p_ent); 720 if (rc) { 721 list_del(&p_ent->list); 722 __qed_spq_return_entry(p_hwfn, p_ent); 723 return rc; 724 } 725 } 726 727 return 0; 728 } 729 730 static int qed_spq_pend_post(struct qed_hwfn *p_hwfn) 731 { 732 struct qed_spq *p_spq = p_hwfn->p_spq; 733 struct qed_spq_entry *p_ent = NULL; 734 735 while (!list_empty(&p_spq->free_pool)) { 736 if (list_empty(&p_spq->unlimited_pending)) 737 break; 738 739 p_ent = list_first_entry(&p_spq->unlimited_pending, 740 struct qed_spq_entry, list); 741 if (!p_ent) 742 return -EINVAL; 743 744 list_del(&p_ent->list); 745 746 qed_spq_add_entry(p_hwfn, p_ent, p_ent->priority); 747 } 748 749 return qed_spq_post_list(p_hwfn, &p_spq->pending, 750 SPQ_HIGH_PRI_RESERVE_DEFAULT); 751 } 752 753 int qed_spq_post(struct qed_hwfn *p_hwfn, 754 struct qed_spq_entry *p_ent, u8 *fw_return_code) 755 { 756 int rc = 0; 757 struct qed_spq *p_spq = p_hwfn ? p_hwfn->p_spq : NULL; 758 bool b_ret_ent = true; 759 760 if (!p_hwfn) 761 return -EINVAL; 762 763 if (!p_ent) { 764 DP_NOTICE(p_hwfn, "Got a NULL pointer\n"); 765 return -EINVAL; 766 } 767 768 /* Complete the entry */ 769 rc = qed_spq_fill_entry(p_hwfn, p_ent); 770 771 spin_lock_bh(&p_spq->lock); 772 773 /* Check return value after LOCK is taken for cleaner error flow */ 774 if (rc) 775 goto spq_post_fail; 776 777 /* Add the request to the pending queue */ 778 rc = qed_spq_add_entry(p_hwfn, p_ent, p_ent->priority); 779 if (rc) 780 goto spq_post_fail; 781 782 rc = qed_spq_pend_post(p_hwfn); 783 if (rc) { 784 /* Since it's possible that pending failed for a different 785 * entry [although unlikely], the failed entry was already 786 * dealt with; No need to return it here. 787 */ 788 b_ret_ent = false; 789 goto spq_post_fail; 790 } 791 792 spin_unlock_bh(&p_spq->lock); 793 794 if (p_ent->comp_mode == QED_SPQ_MODE_EBLOCK) { 795 /* For entries in QED BLOCK mode, the completion code cannot 796 * perform the necessary cleanup - if it did, we couldn't 797 * access p_ent here to see whether it's successful or not. 798 * Thus, after gaining the answer perform the cleanup here. 799 */ 800 rc = qed_spq_block(p_hwfn, p_ent, fw_return_code, 801 p_ent->queue == &p_spq->unlimited_pending); 802 803 if (p_ent->queue == &p_spq->unlimited_pending) { 804 /* This is an allocated p_ent which does not need to 805 * return to pool. 806 */ 807 kfree(p_ent); 808 return rc; 809 } 810 811 if (rc) 812 goto spq_post_fail2; 813 814 /* return to pool */ 815 qed_spq_return_entry(p_hwfn, p_ent); 816 } 817 return rc; 818 819 spq_post_fail2: 820 spin_lock_bh(&p_spq->lock); 821 list_del(&p_ent->list); 822 qed_chain_return_produced(&p_spq->chain); 823 824 spq_post_fail: 825 /* return to the free pool */ 826 if (b_ret_ent) 827 __qed_spq_return_entry(p_hwfn, p_ent); 828 spin_unlock_bh(&p_spq->lock); 829 830 return rc; 831 } 832 833 int qed_spq_completion(struct qed_hwfn *p_hwfn, 834 __le16 echo, 835 u8 fw_return_code, 836 union event_ring_data *p_data) 837 { 838 struct qed_spq *p_spq; 839 struct qed_spq_entry *p_ent = NULL; 840 struct qed_spq_entry *tmp; 841 struct qed_spq_entry *found = NULL; 842 int rc; 843 844 if (!p_hwfn) 845 return -EINVAL; 846 847 p_spq = p_hwfn->p_spq; 848 if (!p_spq) 849 return -EINVAL; 850 851 spin_lock_bh(&p_spq->lock); 852 list_for_each_entry_safe(p_ent, tmp, &p_spq->completion_pending, list) { 853 if (p_ent->elem.hdr.echo == echo) { 854 u16 pos = le16_to_cpu(echo) % SPQ_RING_SIZE; 855 856 list_del(&p_ent->list); 857 858 /* Avoid overriding of SPQ entries when getting 859 * out-of-order completions, by marking the completions 860 * in a bitmap and increasing the chain consumer only 861 * for the first successive completed entries. 862 */ 863 __set_bit(pos, p_spq->p_comp_bitmap); 864 865 while (test_bit(p_spq->comp_bitmap_idx, 866 p_spq->p_comp_bitmap)) { 867 __clear_bit(p_spq->comp_bitmap_idx, 868 p_spq->p_comp_bitmap); 869 p_spq->comp_bitmap_idx++; 870 qed_chain_return_produced(&p_spq->chain); 871 } 872 873 p_spq->comp_count++; 874 found = p_ent; 875 break; 876 } 877 878 /* This is relatively uncommon - depends on scenarios 879 * which have mutliple per-PF sent ramrods. 880 */ 881 DP_VERBOSE(p_hwfn, QED_MSG_SPQ, 882 "Got completion for echo %04x - doesn't match echo %04x in completion pending list\n", 883 le16_to_cpu(echo), 884 le16_to_cpu(p_ent->elem.hdr.echo)); 885 } 886 887 /* Release lock before callback, as callback may post 888 * an additional ramrod. 889 */ 890 spin_unlock_bh(&p_spq->lock); 891 892 if (!found) { 893 DP_NOTICE(p_hwfn, 894 "Failed to find an entry this EQE [echo %04x] completes\n", 895 le16_to_cpu(echo)); 896 return -EEXIST; 897 } 898 899 DP_VERBOSE(p_hwfn, QED_MSG_SPQ, 900 "Complete EQE [echo %04x]: func %p cookie %p)\n", 901 le16_to_cpu(echo), 902 p_ent->comp_cb.function, p_ent->comp_cb.cookie); 903 if (found->comp_cb.function) 904 found->comp_cb.function(p_hwfn, found->comp_cb.cookie, p_data, 905 fw_return_code); 906 else 907 DP_VERBOSE(p_hwfn, 908 QED_MSG_SPQ, 909 "Got a completion without a callback function\n"); 910 911 if ((found->comp_mode != QED_SPQ_MODE_EBLOCK) || 912 (found->queue == &p_spq->unlimited_pending)) 913 /* EBLOCK is responsible for returning its own entry into the 914 * free list, unless it originally added the entry into the 915 * unlimited pending list. 916 */ 917 qed_spq_return_entry(p_hwfn, found); 918 919 /* Attempt to post pending requests */ 920 spin_lock_bh(&p_spq->lock); 921 rc = qed_spq_pend_post(p_hwfn); 922 spin_unlock_bh(&p_spq->lock); 923 924 return rc; 925 } 926 927 struct qed_consq *qed_consq_alloc(struct qed_hwfn *p_hwfn) 928 { 929 struct qed_consq *p_consq; 930 931 /* Allocate ConsQ struct */ 932 p_consq = kzalloc(sizeof(*p_consq), GFP_KERNEL); 933 if (!p_consq) 934 return NULL; 935 936 /* Allocate and initialize EQ chain*/ 937 if (qed_chain_alloc(p_hwfn->cdev, 938 QED_CHAIN_USE_TO_PRODUCE, 939 QED_CHAIN_MODE_PBL, 940 QED_CHAIN_CNT_TYPE_U16, 941 QED_CHAIN_PAGE_SIZE / 0x80, 942 0x80, &p_consq->chain)) 943 goto consq_allocate_fail; 944 945 return p_consq; 946 947 consq_allocate_fail: 948 qed_consq_free(p_hwfn, p_consq); 949 return NULL; 950 } 951 952 void qed_consq_setup(struct qed_hwfn *p_hwfn, struct qed_consq *p_consq) 953 { 954 qed_chain_reset(&p_consq->chain); 955 } 956 957 void qed_consq_free(struct qed_hwfn *p_hwfn, struct qed_consq *p_consq) 958 { 959 if (!p_consq) 960 return; 961 qed_chain_free(p_hwfn->cdev, &p_consq->chain); 962 kfree(p_consq); 963 } 964