1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Huawei HiNIC PCI Express Linux driver 4 * Copyright(c) 2017 Huawei Technologies Co., Ltd 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/types.h> 9 #include <linux/pci.h> 10 #include <linux/device.h> 11 #include <linux/errno.h> 12 #include <linux/slab.h> 13 #include <linux/bitops.h> 14 #include <linux/delay.h> 15 #include <linux/jiffies.h> 16 #include <linux/log2.h> 17 #include <linux/err.h> 18 #include <linux/netdevice.h> 19 #include <net/devlink.h> 20 21 #include "hinic_devlink.h" 22 #include "hinic_sriov.h" 23 #include "hinic_dev.h" 24 #include "hinic_hw_if.h" 25 #include "hinic_hw_eqs.h" 26 #include "hinic_hw_mgmt.h" 27 #include "hinic_hw_qp_ctxt.h" 28 #include "hinic_hw_qp.h" 29 #include "hinic_hw_io.h" 30 #include "hinic_hw_dev.h" 31 32 #define IO_STATUS_TIMEOUT 100 33 #define OUTBOUND_STATE_TIMEOUT 100 34 #define DB_STATE_TIMEOUT 100 35 36 #define MAX_IRQS(max_qps, num_aeqs, num_ceqs) \ 37 (2 * (max_qps) + (num_aeqs) + (num_ceqs)) 38 39 #define ADDR_IN_4BYTES(addr) ((addr) >> 2) 40 41 enum intr_type { 42 INTR_MSIX_TYPE, 43 }; 44 45 enum io_status { 46 IO_STOPPED = 0, 47 IO_RUNNING = 1, 48 }; 49 50 /** 51 * get_capability - convert device capabilities to NIC capabilities 52 * @hwdev: the HW device to set and convert device capabilities for 53 * @dev_cap: device capabilities from FW 54 * 55 * Return 0 - Success, negative - Failure 56 **/ 57 static int parse_capability(struct hinic_hwdev *hwdev, 58 struct hinic_dev_cap *dev_cap) 59 { 60 struct hinic_cap *nic_cap = &hwdev->nic_cap; 61 int num_aeqs, num_ceqs, num_irqs; 62 63 if (!HINIC_IS_VF(hwdev->hwif) && dev_cap->intr_type != INTR_MSIX_TYPE) 64 return -EFAULT; 65 66 num_aeqs = HINIC_HWIF_NUM_AEQS(hwdev->hwif); 67 num_ceqs = HINIC_HWIF_NUM_CEQS(hwdev->hwif); 68 num_irqs = HINIC_HWIF_NUM_IRQS(hwdev->hwif); 69 70 /* Each QP has its own (SQ + RQ) interrupts */ 71 nic_cap->num_qps = (num_irqs - (num_aeqs + num_ceqs)) / 2; 72 73 if (nic_cap->num_qps > HINIC_Q_CTXT_MAX) 74 nic_cap->num_qps = HINIC_Q_CTXT_MAX; 75 76 if (!HINIC_IS_VF(hwdev->hwif)) 77 nic_cap->max_qps = dev_cap->max_sqs + 1; 78 else 79 nic_cap->max_qps = dev_cap->max_sqs; 80 81 if (nic_cap->num_qps > nic_cap->max_qps) 82 nic_cap->num_qps = nic_cap->max_qps; 83 84 if (!HINIC_IS_VF(hwdev->hwif)) { 85 nic_cap->max_vf = dev_cap->max_vf; 86 nic_cap->max_vf_qps = dev_cap->max_vf_sqs + 1; 87 } 88 89 hwdev->port_id = dev_cap->port_id; 90 91 return 0; 92 } 93 94 /** 95 * get_cap_from_fw - get device capabilities from FW 96 * @pfhwdev: the PF HW device to get capabilities for 97 * 98 * Return 0 - Success, negative - Failure 99 **/ 100 static int get_capability(struct hinic_pfhwdev *pfhwdev) 101 { 102 struct hinic_hwdev *hwdev = &pfhwdev->hwdev; 103 struct hinic_hwif *hwif = hwdev->hwif; 104 struct pci_dev *pdev = hwif->pdev; 105 struct hinic_dev_cap dev_cap; 106 u16 out_len; 107 int err; 108 109 out_len = sizeof(dev_cap); 110 111 err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_CFGM, 112 HINIC_CFG_NIC_CAP, &dev_cap, sizeof(dev_cap), 113 &dev_cap, &out_len, HINIC_MGMT_MSG_SYNC); 114 if (err) { 115 dev_err(&pdev->dev, "Failed to get capability from FW\n"); 116 return err; 117 } 118 119 return parse_capability(hwdev, &dev_cap); 120 } 121 122 /** 123 * get_dev_cap - get device capabilities 124 * @hwdev: the NIC HW device to get capabilities for 125 * 126 * Return 0 - Success, negative - Failure 127 **/ 128 static int get_dev_cap(struct hinic_hwdev *hwdev) 129 { 130 struct hinic_hwif *hwif = hwdev->hwif; 131 struct pci_dev *pdev = hwif->pdev; 132 struct hinic_pfhwdev *pfhwdev; 133 int err; 134 135 switch (HINIC_FUNC_TYPE(hwif)) { 136 case HINIC_PPF: 137 case HINIC_PF: 138 case HINIC_VF: 139 pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); 140 err = get_capability(pfhwdev); 141 if (err) { 142 dev_err(&pdev->dev, "Failed to get capability\n"); 143 return err; 144 } 145 break; 146 default: 147 dev_err(&pdev->dev, "Unsupported PCI Function type\n"); 148 return -EINVAL; 149 } 150 151 return 0; 152 } 153 154 /** 155 * init_msix - enable the msix and save the entries 156 * @hwdev: the NIC HW device 157 * 158 * Return 0 - Success, negative - Failure 159 **/ 160 static int init_msix(struct hinic_hwdev *hwdev) 161 { 162 struct hinic_hwif *hwif = hwdev->hwif; 163 struct pci_dev *pdev = hwif->pdev; 164 int nr_irqs, num_aeqs, num_ceqs; 165 size_t msix_entries_size; 166 int i, err; 167 168 num_aeqs = HINIC_HWIF_NUM_AEQS(hwif); 169 num_ceqs = HINIC_HWIF_NUM_CEQS(hwif); 170 nr_irqs = MAX_IRQS(HINIC_MAX_QPS, num_aeqs, num_ceqs); 171 if (nr_irqs > HINIC_HWIF_NUM_IRQS(hwif)) 172 nr_irqs = HINIC_HWIF_NUM_IRQS(hwif); 173 174 msix_entries_size = nr_irqs * sizeof(*hwdev->msix_entries); 175 hwdev->msix_entries = devm_kzalloc(&pdev->dev, msix_entries_size, 176 GFP_KERNEL); 177 if (!hwdev->msix_entries) 178 return -ENOMEM; 179 180 for (i = 0; i < nr_irqs; i++) 181 hwdev->msix_entries[i].entry = i; 182 183 err = pci_enable_msix_exact(pdev, hwdev->msix_entries, nr_irqs); 184 if (err) { 185 dev_err(&pdev->dev, "Failed to enable pci msix\n"); 186 return err; 187 } 188 189 return 0; 190 } 191 192 /** 193 * disable_msix - disable the msix 194 * @hwdev: the NIC HW device 195 **/ 196 static void disable_msix(struct hinic_hwdev *hwdev) 197 { 198 struct hinic_hwif *hwif = hwdev->hwif; 199 struct pci_dev *pdev = hwif->pdev; 200 201 pci_disable_msix(pdev); 202 } 203 204 /** 205 * hinic_port_msg_cmd - send port msg to mgmt 206 * @hwdev: the NIC HW device 207 * @cmd: the port command 208 * @buf_in: input buffer 209 * @in_size: input size 210 * @buf_out: output buffer 211 * @out_size: returned output size 212 * 213 * Return 0 - Success, negative - Failure 214 **/ 215 int hinic_port_msg_cmd(struct hinic_hwdev *hwdev, enum hinic_port_cmd cmd, 216 void *buf_in, u16 in_size, void *buf_out, u16 *out_size) 217 { 218 struct hinic_pfhwdev *pfhwdev; 219 220 pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); 221 222 return hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_L2NIC, cmd, 223 buf_in, in_size, buf_out, out_size, 224 HINIC_MGMT_MSG_SYNC); 225 } 226 227 int hinic_hilink_msg_cmd(struct hinic_hwdev *hwdev, enum hinic_hilink_cmd cmd, 228 void *buf_in, u16 in_size, void *buf_out, 229 u16 *out_size) 230 { 231 struct hinic_pfhwdev *pfhwdev; 232 233 pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); 234 235 return hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_HILINK, cmd, 236 buf_in, in_size, buf_out, out_size, 237 HINIC_MGMT_MSG_SYNC); 238 } 239 240 /** 241 * init_fw_ctxt- Init Firmware tables before network mgmt and io operations 242 * @hwdev: the NIC HW device 243 * 244 * Return 0 - Success, negative - Failure 245 **/ 246 static int init_fw_ctxt(struct hinic_hwdev *hwdev) 247 { 248 struct hinic_hwif *hwif = hwdev->hwif; 249 struct pci_dev *pdev = hwif->pdev; 250 struct hinic_cmd_fw_ctxt fw_ctxt; 251 u16 out_size = sizeof(fw_ctxt); 252 int err; 253 254 fw_ctxt.func_idx = HINIC_HWIF_FUNC_IDX(hwif); 255 fw_ctxt.rx_buf_sz = HINIC_RX_BUF_SZ; 256 257 err = hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_FWCTXT_INIT, 258 &fw_ctxt, sizeof(fw_ctxt), 259 &fw_ctxt, &out_size); 260 if (err || (out_size != sizeof(fw_ctxt)) || fw_ctxt.status) { 261 dev_err(&pdev->dev, "Failed to init FW ctxt, err: %d, status: 0x%x, out size: 0x%x\n", 262 err, fw_ctxt.status, out_size); 263 return -EIO; 264 } 265 266 return 0; 267 } 268 269 /** 270 * set_hw_ioctxt - set the shape of the IO queues in FW 271 * @hwdev: the NIC HW device 272 * @rq_depth: rq depth 273 * @sq_depth: sq depth 274 * 275 * Return 0 - Success, negative - Failure 276 **/ 277 static int set_hw_ioctxt(struct hinic_hwdev *hwdev, unsigned int sq_depth, 278 unsigned int rq_depth) 279 { 280 struct hinic_hwif *hwif = hwdev->hwif; 281 struct hinic_cmd_hw_ioctxt hw_ioctxt; 282 struct hinic_pfhwdev *pfhwdev; 283 284 hw_ioctxt.func_idx = HINIC_HWIF_FUNC_IDX(hwif); 285 hw_ioctxt.ppf_idx = HINIC_HWIF_PPF_IDX(hwif); 286 287 hw_ioctxt.set_cmdq_depth = HW_IOCTXT_SET_CMDQ_DEPTH_DEFAULT; 288 hw_ioctxt.cmdq_depth = 0; 289 290 hw_ioctxt.lro_en = 1; 291 292 hw_ioctxt.rq_depth = ilog2(rq_depth); 293 294 hw_ioctxt.rx_buf_sz_idx = HINIC_RX_BUF_SZ_IDX; 295 296 hw_ioctxt.sq_depth = ilog2(sq_depth); 297 298 pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); 299 300 return hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM, 301 HINIC_COMM_CMD_HWCTXT_SET, 302 &hw_ioctxt, sizeof(hw_ioctxt), NULL, 303 NULL, HINIC_MGMT_MSG_SYNC); 304 } 305 306 static int wait_for_outbound_state(struct hinic_hwdev *hwdev) 307 { 308 enum hinic_outbound_state outbound_state; 309 struct hinic_hwif *hwif = hwdev->hwif; 310 struct pci_dev *pdev = hwif->pdev; 311 unsigned long end; 312 313 end = jiffies + msecs_to_jiffies(OUTBOUND_STATE_TIMEOUT); 314 do { 315 outbound_state = hinic_outbound_state_get(hwif); 316 317 if (outbound_state == HINIC_OUTBOUND_ENABLE) 318 return 0; 319 320 msleep(20); 321 } while (time_before(jiffies, end)); 322 323 dev_err(&pdev->dev, "Wait for OUTBOUND - Timeout\n"); 324 return -EFAULT; 325 } 326 327 static int wait_for_db_state(struct hinic_hwdev *hwdev) 328 { 329 struct hinic_hwif *hwif = hwdev->hwif; 330 struct pci_dev *pdev = hwif->pdev; 331 enum hinic_db_state db_state; 332 unsigned long end; 333 334 end = jiffies + msecs_to_jiffies(DB_STATE_TIMEOUT); 335 do { 336 db_state = hinic_db_state_get(hwif); 337 338 if (db_state == HINIC_DB_ENABLE) 339 return 0; 340 341 msleep(20); 342 } while (time_before(jiffies, end)); 343 344 dev_err(&pdev->dev, "Wait for DB - Timeout\n"); 345 return -EFAULT; 346 } 347 348 /** 349 * clear_io_resource - set the IO resources as not active in the NIC 350 * @hwdev: the NIC HW device 351 * 352 * Return 0 - Success, negative - Failure 353 **/ 354 static int clear_io_resources(struct hinic_hwdev *hwdev) 355 { 356 struct hinic_cmd_clear_io_res cmd_clear_io_res; 357 struct hinic_hwif *hwif = hwdev->hwif; 358 struct pci_dev *pdev = hwif->pdev; 359 struct hinic_pfhwdev *pfhwdev; 360 int err; 361 362 /* sleep 100ms to wait for firmware stopping I/O */ 363 msleep(100); 364 365 cmd_clear_io_res.func_idx = HINIC_HWIF_FUNC_IDX(hwif); 366 367 pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); 368 369 err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM, 370 HINIC_COMM_CMD_IO_RES_CLEAR, &cmd_clear_io_res, 371 sizeof(cmd_clear_io_res), NULL, NULL, 372 HINIC_MGMT_MSG_SYNC); 373 if (err) { 374 dev_err(&pdev->dev, "Failed to clear IO resources\n"); 375 return err; 376 } 377 378 return 0; 379 } 380 381 /** 382 * set_resources_state - set the state of the resources in the NIC 383 * @hwdev: the NIC HW device 384 * @state: the state to set 385 * 386 * Return 0 - Success, negative - Failure 387 **/ 388 static int set_resources_state(struct hinic_hwdev *hwdev, 389 enum hinic_res_state state) 390 { 391 struct hinic_cmd_set_res_state res_state; 392 struct hinic_hwif *hwif = hwdev->hwif; 393 struct hinic_pfhwdev *pfhwdev; 394 395 res_state.func_idx = HINIC_HWIF_FUNC_IDX(hwif); 396 res_state.state = state; 397 398 pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); 399 400 return hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, 401 HINIC_MOD_COMM, 402 HINIC_COMM_CMD_RES_STATE_SET, 403 &res_state, sizeof(res_state), NULL, 404 NULL, HINIC_MGMT_MSG_SYNC); 405 } 406 407 /** 408 * get_base_qpn - get the first qp number 409 * @hwdev: the NIC HW device 410 * @base_qpn: returned qp number 411 * 412 * Return 0 - Success, negative - Failure 413 **/ 414 static int get_base_qpn(struct hinic_hwdev *hwdev, u16 *base_qpn) 415 { 416 struct hinic_cmd_base_qpn cmd_base_qpn; 417 struct hinic_hwif *hwif = hwdev->hwif; 418 u16 out_size = sizeof(cmd_base_qpn); 419 struct pci_dev *pdev = hwif->pdev; 420 int err; 421 422 cmd_base_qpn.func_idx = HINIC_HWIF_FUNC_IDX(hwif); 423 424 err = hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_GET_GLOBAL_QPN, 425 &cmd_base_qpn, sizeof(cmd_base_qpn), 426 &cmd_base_qpn, &out_size); 427 if (err || (out_size != sizeof(cmd_base_qpn)) || cmd_base_qpn.status) { 428 dev_err(&pdev->dev, "Failed to get base qpn, err: %d, status: 0x%x, out size: 0x%x\n", 429 err, cmd_base_qpn.status, out_size); 430 return -EIO; 431 } 432 433 *base_qpn = cmd_base_qpn.qpn; 434 return 0; 435 } 436 437 /** 438 * hinic_hwdev_ifup - Preparing the HW for passing IO 439 * @hwdev: the NIC HW device 440 * @sq_depth: the send queue depth 441 * @rq_depth: the receive queue depth 442 * 443 * Return 0 - Success, negative - Failure 444 **/ 445 int hinic_hwdev_ifup(struct hinic_hwdev *hwdev, u16 sq_depth, u16 rq_depth) 446 { 447 struct hinic_func_to_io *func_to_io = &hwdev->func_to_io; 448 struct hinic_cap *nic_cap = &hwdev->nic_cap; 449 struct hinic_hwif *hwif = hwdev->hwif; 450 int err, num_aeqs, num_ceqs, num_qps; 451 struct msix_entry *ceq_msix_entries; 452 struct msix_entry *sq_msix_entries; 453 struct msix_entry *rq_msix_entries; 454 struct pci_dev *pdev = hwif->pdev; 455 u16 base_qpn; 456 457 err = get_base_qpn(hwdev, &base_qpn); 458 if (err) { 459 dev_err(&pdev->dev, "Failed to get global base qp number\n"); 460 return err; 461 } 462 463 num_aeqs = HINIC_HWIF_NUM_AEQS(hwif); 464 num_ceqs = HINIC_HWIF_NUM_CEQS(hwif); 465 466 ceq_msix_entries = &hwdev->msix_entries[num_aeqs]; 467 func_to_io->hwdev = hwdev; 468 func_to_io->sq_depth = sq_depth; 469 func_to_io->rq_depth = rq_depth; 470 func_to_io->global_qpn = base_qpn; 471 472 err = hinic_io_init(func_to_io, hwif, nic_cap->max_qps, num_ceqs, 473 ceq_msix_entries); 474 if (err) { 475 dev_err(&pdev->dev, "Failed to init IO channel\n"); 476 return err; 477 } 478 479 num_qps = nic_cap->num_qps; 480 sq_msix_entries = &hwdev->msix_entries[num_aeqs + num_ceqs]; 481 rq_msix_entries = &hwdev->msix_entries[num_aeqs + num_ceqs + num_qps]; 482 483 err = hinic_io_create_qps(func_to_io, base_qpn, num_qps, 484 sq_msix_entries, rq_msix_entries); 485 if (err) { 486 dev_err(&pdev->dev, "Failed to create QPs\n"); 487 goto err_create_qps; 488 } 489 490 err = wait_for_db_state(hwdev); 491 if (err) { 492 dev_warn(&pdev->dev, "db - disabled, try again\n"); 493 hinic_db_state_set(hwif, HINIC_DB_ENABLE); 494 } 495 496 err = set_hw_ioctxt(hwdev, sq_depth, rq_depth); 497 if (err) { 498 dev_err(&pdev->dev, "Failed to set HW IO ctxt\n"); 499 goto err_hw_ioctxt; 500 } 501 502 return 0; 503 504 err_hw_ioctxt: 505 hinic_io_destroy_qps(func_to_io, num_qps); 506 507 err_create_qps: 508 hinic_io_free(func_to_io); 509 return err; 510 } 511 512 /** 513 * hinic_hwdev_ifdown - Closing the HW for passing IO 514 * @hwdev: the NIC HW device 515 * 516 **/ 517 void hinic_hwdev_ifdown(struct hinic_hwdev *hwdev) 518 { 519 struct hinic_func_to_io *func_to_io = &hwdev->func_to_io; 520 struct hinic_cap *nic_cap = &hwdev->nic_cap; 521 522 clear_io_resources(hwdev); 523 524 hinic_io_destroy_qps(func_to_io, nic_cap->num_qps); 525 hinic_io_free(func_to_io); 526 } 527 528 /** 529 * hinic_hwdev_cb_register - register callback handler for MGMT events 530 * @hwdev: the NIC HW device 531 * @cmd: the mgmt event 532 * @handle: private data for the handler 533 * @handler: event handler 534 **/ 535 void hinic_hwdev_cb_register(struct hinic_hwdev *hwdev, 536 enum hinic_mgmt_msg_cmd cmd, void *handle, 537 void (*handler)(void *handle, void *buf_in, 538 u16 in_size, void *buf_out, 539 u16 *out_size)) 540 { 541 struct hinic_pfhwdev *pfhwdev; 542 struct hinic_nic_cb *nic_cb; 543 u8 cmd_cb; 544 545 pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); 546 547 cmd_cb = cmd - HINIC_MGMT_MSG_CMD_BASE; 548 nic_cb = &pfhwdev->nic_cb[cmd_cb]; 549 550 nic_cb->handler = handler; 551 nic_cb->handle = handle; 552 nic_cb->cb_state = HINIC_CB_ENABLED; 553 } 554 555 /** 556 * hinic_hwdev_cb_unregister - unregister callback handler for MGMT events 557 * @hwdev: the NIC HW device 558 * @cmd: the mgmt event 559 **/ 560 void hinic_hwdev_cb_unregister(struct hinic_hwdev *hwdev, 561 enum hinic_mgmt_msg_cmd cmd) 562 { 563 struct hinic_hwif *hwif = hwdev->hwif; 564 struct hinic_pfhwdev *pfhwdev; 565 struct hinic_nic_cb *nic_cb; 566 u8 cmd_cb; 567 568 if (!HINIC_IS_PF(hwif) && !HINIC_IS_PPF(hwif)) 569 return; 570 571 pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); 572 573 cmd_cb = cmd - HINIC_MGMT_MSG_CMD_BASE; 574 nic_cb = &pfhwdev->nic_cb[cmd_cb]; 575 576 nic_cb->cb_state &= ~HINIC_CB_ENABLED; 577 578 while (nic_cb->cb_state & HINIC_CB_RUNNING) 579 schedule(); 580 581 nic_cb->handler = NULL; 582 } 583 584 /** 585 * nic_mgmt_msg_handler - nic mgmt event handler 586 * @handle: private data for the handler 587 * @cmd: message command 588 * @buf_in: input buffer 589 * @in_size: input size 590 * @buf_out: output buffer 591 * @out_size: returned output size 592 **/ 593 static void nic_mgmt_msg_handler(void *handle, u8 cmd, void *buf_in, 594 u16 in_size, void *buf_out, u16 *out_size) 595 { 596 struct hinic_pfhwdev *pfhwdev = handle; 597 enum hinic_cb_state cb_state; 598 struct hinic_nic_cb *nic_cb; 599 struct hinic_hwdev *hwdev; 600 struct hinic_hwif *hwif; 601 struct pci_dev *pdev; 602 u8 cmd_cb; 603 604 hwdev = &pfhwdev->hwdev; 605 hwif = hwdev->hwif; 606 pdev = hwif->pdev; 607 608 if ((cmd < HINIC_MGMT_MSG_CMD_BASE) || 609 (cmd >= HINIC_MGMT_MSG_CMD_MAX)) { 610 dev_err(&pdev->dev, "unknown L2NIC event, cmd = %d\n", cmd); 611 return; 612 } 613 614 cmd_cb = cmd - HINIC_MGMT_MSG_CMD_BASE; 615 616 nic_cb = &pfhwdev->nic_cb[cmd_cb]; 617 618 cb_state = cmpxchg(&nic_cb->cb_state, 619 HINIC_CB_ENABLED, 620 HINIC_CB_ENABLED | HINIC_CB_RUNNING); 621 622 if ((cb_state == HINIC_CB_ENABLED) && (nic_cb->handler)) 623 nic_cb->handler(nic_cb->handle, buf_in, 624 in_size, buf_out, out_size); 625 else 626 dev_err(&pdev->dev, "Unhandled NIC Event %d\n", cmd); 627 628 nic_cb->cb_state &= ~HINIC_CB_RUNNING; 629 } 630 631 static void hinic_comm_recv_mgmt_self_cmd_reg(struct hinic_pfhwdev *pfhwdev, 632 u8 cmd, 633 comm_mgmt_self_msg_proc proc) 634 { 635 u8 cmd_idx; 636 637 cmd_idx = pfhwdev->proc.cmd_num; 638 if (cmd_idx >= HINIC_COMM_SELF_CMD_MAX) { 639 dev_err(&pfhwdev->hwdev.hwif->pdev->dev, 640 "Register recv mgmt process failed, cmd: 0x%x\n", cmd); 641 return; 642 } 643 644 pfhwdev->proc.info[cmd_idx].cmd = cmd; 645 pfhwdev->proc.info[cmd_idx].proc = proc; 646 pfhwdev->proc.cmd_num++; 647 } 648 649 static void hinic_comm_recv_mgmt_self_cmd_unreg(struct hinic_pfhwdev *pfhwdev, 650 u8 cmd) 651 { 652 u8 cmd_idx; 653 654 cmd_idx = pfhwdev->proc.cmd_num; 655 if (cmd_idx >= HINIC_COMM_SELF_CMD_MAX) { 656 dev_err(&pfhwdev->hwdev.hwif->pdev->dev, "Unregister recv mgmt process failed, cmd: 0x%x\n", 657 cmd); 658 return; 659 } 660 661 for (cmd_idx = 0; cmd_idx < HINIC_COMM_SELF_CMD_MAX; cmd_idx++) { 662 if (cmd == pfhwdev->proc.info[cmd_idx].cmd) { 663 pfhwdev->proc.info[cmd_idx].cmd = 0; 664 pfhwdev->proc.info[cmd_idx].proc = NULL; 665 pfhwdev->proc.cmd_num--; 666 } 667 } 668 } 669 670 static void comm_mgmt_msg_handler(void *handle, u8 cmd, void *buf_in, 671 u16 in_size, void *buf_out, u16 *out_size) 672 { 673 struct hinic_pfhwdev *pfhwdev = handle; 674 u8 cmd_idx; 675 676 for (cmd_idx = 0; cmd_idx < pfhwdev->proc.cmd_num; cmd_idx++) { 677 if (cmd == pfhwdev->proc.info[cmd_idx].cmd) { 678 if (!pfhwdev->proc.info[cmd_idx].proc) { 679 dev_warn(&pfhwdev->hwdev.hwif->pdev->dev, 680 "PF recv mgmt comm msg handle null, cmd: 0x%x\n", 681 cmd); 682 } else { 683 pfhwdev->proc.info[cmd_idx].proc 684 (&pfhwdev->hwdev, buf_in, in_size, 685 buf_out, out_size); 686 } 687 688 return; 689 } 690 } 691 692 dev_warn(&pfhwdev->hwdev.hwif->pdev->dev, "Received unknown mgmt cpu event: 0x%x\n", 693 cmd); 694 695 *out_size = 0; 696 } 697 698 /* pf fault report event */ 699 static void pf_fault_event_handler(void *dev, void *buf_in, u16 in_size, 700 void *buf_out, u16 *out_size) 701 { 702 struct hinic_cmd_fault_event *fault_event = buf_in; 703 struct hinic_hwdev *hwdev = dev; 704 705 if (in_size != sizeof(*fault_event)) { 706 dev_err(&hwdev->hwif->pdev->dev, "Invalid fault event report, length: %d, should be %zu\n", 707 in_size, sizeof(*fault_event)); 708 return; 709 } 710 711 if (!hwdev->devlink_dev || IS_ERR_OR_NULL(hwdev->devlink_dev->hw_fault_reporter)) 712 return; 713 714 devlink_health_report(hwdev->devlink_dev->hw_fault_reporter, 715 "HW fatal error reported", &fault_event->event); 716 } 717 718 static void mgmt_watchdog_timeout_event_handler(void *dev, 719 void *buf_in, u16 in_size, 720 void *buf_out, u16 *out_size) 721 { 722 struct hinic_mgmt_watchdog_info *watchdog_info = buf_in; 723 struct hinic_hwdev *hwdev = dev; 724 725 if (in_size != sizeof(*watchdog_info)) { 726 dev_err(&hwdev->hwif->pdev->dev, "Invalid mgmt watchdog report, length: %d, should be %zu\n", 727 in_size, sizeof(*watchdog_info)); 728 return; 729 } 730 731 if (!hwdev->devlink_dev || IS_ERR_OR_NULL(hwdev->devlink_dev->fw_fault_reporter)) 732 return; 733 734 devlink_health_report(hwdev->devlink_dev->fw_fault_reporter, 735 "FW fatal error reported", watchdog_info); 736 } 737 738 /** 739 * init_pfhwdev - Initialize the extended components of PF 740 * @pfhwdev: the HW device for PF 741 * 742 * Return 0 - success, negative - failure 743 **/ 744 static int init_pfhwdev(struct hinic_pfhwdev *pfhwdev) 745 { 746 struct hinic_hwdev *hwdev = &pfhwdev->hwdev; 747 struct hinic_hwif *hwif = hwdev->hwif; 748 struct pci_dev *pdev = hwif->pdev; 749 int err; 750 751 err = hinic_pf_to_mgmt_init(&pfhwdev->pf_to_mgmt, hwif); 752 if (err) { 753 dev_err(&pdev->dev, "Failed to initialize PF to MGMT channel\n"); 754 return err; 755 } 756 757 err = hinic_devlink_register(hwdev->devlink_dev, &pdev->dev); 758 if (err) { 759 dev_err(&hwif->pdev->dev, "Failed to register devlink\n"); 760 hinic_pf_to_mgmt_free(&pfhwdev->pf_to_mgmt); 761 return err; 762 } 763 764 err = hinic_func_to_func_init(hwdev); 765 if (err) { 766 dev_err(&hwif->pdev->dev, "Failed to init mailbox\n"); 767 hinic_devlink_unregister(hwdev->devlink_dev); 768 hinic_pf_to_mgmt_free(&pfhwdev->pf_to_mgmt); 769 return err; 770 } 771 772 if (!HINIC_IS_VF(hwif)) { 773 hinic_register_mgmt_msg_cb(&pfhwdev->pf_to_mgmt, 774 HINIC_MOD_L2NIC, pfhwdev, 775 nic_mgmt_msg_handler); 776 hinic_register_mgmt_msg_cb(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM, 777 pfhwdev, comm_mgmt_msg_handler); 778 hinic_comm_recv_mgmt_self_cmd_reg(pfhwdev, 779 HINIC_COMM_CMD_FAULT_REPORT, 780 pf_fault_event_handler); 781 hinic_comm_recv_mgmt_self_cmd_reg 782 (pfhwdev, HINIC_COMM_CMD_WATCHDOG_INFO, 783 mgmt_watchdog_timeout_event_handler); 784 } else { 785 hinic_register_vf_mbox_cb(hwdev, HINIC_MOD_L2NIC, 786 nic_mgmt_msg_handler); 787 } 788 789 hinic_set_pf_action(hwif, HINIC_PF_MGMT_ACTIVE); 790 791 return 0; 792 } 793 794 /** 795 * free_pfhwdev - Free the extended components of PF 796 * @pfhwdev: the HW device for PF 797 **/ 798 static void free_pfhwdev(struct hinic_pfhwdev *pfhwdev) 799 { 800 struct hinic_hwdev *hwdev = &pfhwdev->hwdev; 801 802 hinic_set_pf_action(hwdev->hwif, HINIC_PF_MGMT_INIT); 803 804 if (!HINIC_IS_VF(hwdev->hwif)) { 805 hinic_comm_recv_mgmt_self_cmd_unreg(pfhwdev, 806 HINIC_COMM_CMD_WATCHDOG_INFO); 807 hinic_comm_recv_mgmt_self_cmd_unreg(pfhwdev, 808 HINIC_COMM_CMD_FAULT_REPORT); 809 hinic_unregister_mgmt_msg_cb(&pfhwdev->pf_to_mgmt, 810 HINIC_MOD_COMM); 811 hinic_unregister_mgmt_msg_cb(&pfhwdev->pf_to_mgmt, 812 HINIC_MOD_L2NIC); 813 } else { 814 hinic_unregister_vf_mbox_cb(hwdev, HINIC_MOD_L2NIC); 815 } 816 817 hinic_func_to_func_free(hwdev); 818 819 hinic_devlink_unregister(hwdev->devlink_dev); 820 821 hinic_pf_to_mgmt_free(&pfhwdev->pf_to_mgmt); 822 } 823 824 static int hinic_l2nic_reset(struct hinic_hwdev *hwdev) 825 { 826 struct hinic_cmd_l2nic_reset l2nic_reset = {0}; 827 u16 out_size = sizeof(l2nic_reset); 828 struct hinic_pfhwdev *pfhwdev; 829 int err; 830 831 pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); 832 833 l2nic_reset.func_id = HINIC_HWIF_FUNC_IDX(hwdev->hwif); 834 /* 0 represents standard l2nic reset flow */ 835 l2nic_reset.reset_flag = 0; 836 837 err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM, 838 HINIC_COMM_CMD_L2NIC_RESET, &l2nic_reset, 839 sizeof(l2nic_reset), &l2nic_reset, 840 &out_size, HINIC_MGMT_MSG_SYNC); 841 if (err || !out_size || l2nic_reset.status) { 842 dev_err(&hwdev->hwif->pdev->dev, "Failed to reset L2NIC resources, err: %d, status: 0x%x, out_size: 0x%x\n", 843 err, l2nic_reset.status, out_size); 844 return -EIO; 845 } 846 847 return 0; 848 } 849 850 int hinic_get_interrupt_cfg(struct hinic_hwdev *hwdev, 851 struct hinic_msix_config *interrupt_info) 852 { 853 u16 out_size = sizeof(*interrupt_info); 854 struct hinic_pfhwdev *pfhwdev; 855 int err; 856 857 if (!hwdev || !interrupt_info) 858 return -EINVAL; 859 860 pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); 861 862 interrupt_info->func_id = HINIC_HWIF_FUNC_IDX(hwdev->hwif); 863 864 err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM, 865 HINIC_COMM_CMD_MSI_CTRL_REG_RD_BY_UP, 866 interrupt_info, sizeof(*interrupt_info), 867 interrupt_info, &out_size, HINIC_MGMT_MSG_SYNC); 868 if (err || !out_size || interrupt_info->status) { 869 dev_err(&hwdev->hwif->pdev->dev, "Failed to get interrupt config, err: %d, status: 0x%x, out size: 0x%x\n", 870 err, interrupt_info->status, out_size); 871 return -EIO; 872 } 873 874 return 0; 875 } 876 877 int hinic_set_interrupt_cfg(struct hinic_hwdev *hwdev, 878 struct hinic_msix_config *interrupt_info) 879 { 880 u16 out_size = sizeof(*interrupt_info); 881 struct hinic_msix_config temp_info; 882 struct hinic_pfhwdev *pfhwdev; 883 int err; 884 885 if (!hwdev) 886 return -EINVAL; 887 888 pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); 889 890 interrupt_info->func_id = HINIC_HWIF_FUNC_IDX(hwdev->hwif); 891 892 err = hinic_get_interrupt_cfg(hwdev, &temp_info); 893 if (err) 894 return -EINVAL; 895 896 interrupt_info->lli_credit_cnt = temp_info.lli_timer_cnt; 897 interrupt_info->lli_timer_cnt = temp_info.lli_timer_cnt; 898 899 err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM, 900 HINIC_COMM_CMD_MSI_CTRL_REG_WR_BY_UP, 901 interrupt_info, sizeof(*interrupt_info), 902 interrupt_info, &out_size, HINIC_MGMT_MSG_SYNC); 903 if (err || !out_size || interrupt_info->status) { 904 dev_err(&hwdev->hwif->pdev->dev, "Failed to get interrupt config, err: %d, status: 0x%x, out size: 0x%x\n", 905 err, interrupt_info->status, out_size); 906 return -EIO; 907 } 908 909 return 0; 910 } 911 912 /** 913 * hinic_init_hwdev - Initialize the NIC HW 914 * @pdev: the NIC pci device 915 * @devlink: the poniter of hinic devlink 916 * 917 * Return initialized NIC HW device 918 * 919 * Initialize the NIC HW device and return a pointer to it 920 **/ 921 struct hinic_hwdev *hinic_init_hwdev(struct pci_dev *pdev, struct devlink *devlink) 922 { 923 struct hinic_pfhwdev *pfhwdev; 924 struct hinic_hwdev *hwdev; 925 struct hinic_hwif *hwif; 926 int err, num_aeqs; 927 928 hwif = devm_kzalloc(&pdev->dev, sizeof(*hwif), GFP_KERNEL); 929 if (!hwif) 930 return ERR_PTR(-ENOMEM); 931 932 err = hinic_init_hwif(hwif, pdev); 933 if (err) { 934 dev_err(&pdev->dev, "Failed to init HW interface\n"); 935 return ERR_PTR(err); 936 } 937 938 pfhwdev = devm_kzalloc(&pdev->dev, sizeof(*pfhwdev), GFP_KERNEL); 939 if (!pfhwdev) { 940 err = -ENOMEM; 941 goto err_pfhwdev_alloc; 942 } 943 944 hwdev = &pfhwdev->hwdev; 945 hwdev->hwif = hwif; 946 hwdev->devlink_dev = devlink_priv(devlink); 947 hwdev->devlink_dev->hwdev = hwdev; 948 949 err = init_msix(hwdev); 950 if (err) { 951 dev_err(&pdev->dev, "Failed to init msix\n"); 952 goto err_init_msix; 953 } 954 955 err = wait_for_outbound_state(hwdev); 956 if (err) { 957 dev_warn(&pdev->dev, "outbound - disabled, try again\n"); 958 hinic_outbound_state_set(hwif, HINIC_OUTBOUND_ENABLE); 959 } 960 961 num_aeqs = HINIC_HWIF_NUM_AEQS(hwif); 962 963 err = hinic_aeqs_init(&hwdev->aeqs, hwif, num_aeqs, 964 HINIC_DEFAULT_AEQ_LEN, HINIC_EQ_PAGE_SIZE, 965 hwdev->msix_entries); 966 if (err) { 967 dev_err(&pdev->dev, "Failed to init async event queues\n"); 968 goto err_aeqs_init; 969 } 970 971 err = init_pfhwdev(pfhwdev); 972 if (err) { 973 dev_err(&pdev->dev, "Failed to init PF HW device\n"); 974 goto err_init_pfhwdev; 975 } 976 977 err = hinic_l2nic_reset(hwdev); 978 if (err) 979 goto err_l2nic_reset; 980 981 err = get_dev_cap(hwdev); 982 if (err) { 983 dev_err(&pdev->dev, "Failed to get device capabilities\n"); 984 goto err_dev_cap; 985 } 986 987 mutex_init(&hwdev->func_to_io.nic_cfg.cfg_mutex); 988 989 err = hinic_vf_func_init(hwdev); 990 if (err) { 991 dev_err(&pdev->dev, "Failed to init nic mbox\n"); 992 goto err_vf_func_init; 993 } 994 995 err = init_fw_ctxt(hwdev); 996 if (err) { 997 dev_err(&pdev->dev, "Failed to init function table\n"); 998 goto err_init_fw_ctxt; 999 } 1000 1001 err = set_resources_state(hwdev, HINIC_RES_ACTIVE); 1002 if (err) { 1003 dev_err(&pdev->dev, "Failed to set resources state\n"); 1004 goto err_resources_state; 1005 } 1006 1007 return hwdev; 1008 1009 err_resources_state: 1010 err_init_fw_ctxt: 1011 hinic_vf_func_free(hwdev); 1012 err_vf_func_init: 1013 err_l2nic_reset: 1014 err_dev_cap: 1015 free_pfhwdev(pfhwdev); 1016 1017 err_init_pfhwdev: 1018 hinic_aeqs_free(&hwdev->aeqs); 1019 1020 err_aeqs_init: 1021 disable_msix(hwdev); 1022 1023 err_init_msix: 1024 err_pfhwdev_alloc: 1025 hinic_free_hwif(hwif); 1026 if (err > 0) 1027 err = -EIO; 1028 return ERR_PTR(err); 1029 } 1030 1031 /** 1032 * hinic_free_hwdev - Free the NIC HW device 1033 * @hwdev: the NIC HW device 1034 **/ 1035 void hinic_free_hwdev(struct hinic_hwdev *hwdev) 1036 { 1037 struct hinic_pfhwdev *pfhwdev = container_of(hwdev, 1038 struct hinic_pfhwdev, 1039 hwdev); 1040 1041 set_resources_state(hwdev, HINIC_RES_CLEAN); 1042 1043 hinic_vf_func_free(hwdev); 1044 1045 free_pfhwdev(pfhwdev); 1046 1047 hinic_aeqs_free(&hwdev->aeqs); 1048 1049 disable_msix(hwdev); 1050 1051 hinic_free_hwif(hwdev->hwif); 1052 } 1053 1054 int hinic_hwdev_max_num_qps(struct hinic_hwdev *hwdev) 1055 { 1056 struct hinic_cap *nic_cap = &hwdev->nic_cap; 1057 1058 return nic_cap->max_qps; 1059 } 1060 1061 /** 1062 * hinic_hwdev_num_qps - return the number QPs available for use 1063 * @hwdev: the NIC HW device 1064 * 1065 * Return number QPs available for use 1066 **/ 1067 int hinic_hwdev_num_qps(struct hinic_hwdev *hwdev) 1068 { 1069 struct hinic_cap *nic_cap = &hwdev->nic_cap; 1070 1071 return nic_cap->num_qps; 1072 } 1073 1074 /** 1075 * hinic_hwdev_get_sq - get SQ 1076 * @hwdev: the NIC HW device 1077 * @i: the position of the SQ 1078 * 1079 * Return: the SQ in the i position 1080 **/ 1081 struct hinic_sq *hinic_hwdev_get_sq(struct hinic_hwdev *hwdev, int i) 1082 { 1083 struct hinic_func_to_io *func_to_io = &hwdev->func_to_io; 1084 struct hinic_qp *qp = &func_to_io->qps[i]; 1085 1086 if (i >= hinic_hwdev_num_qps(hwdev)) 1087 return NULL; 1088 1089 return &qp->sq; 1090 } 1091 1092 /** 1093 * hinic_hwdev_get_sq - get RQ 1094 * @hwdev: the NIC HW device 1095 * @i: the position of the RQ 1096 * 1097 * Return: the RQ in the i position 1098 **/ 1099 struct hinic_rq *hinic_hwdev_get_rq(struct hinic_hwdev *hwdev, int i) 1100 { 1101 struct hinic_func_to_io *func_to_io = &hwdev->func_to_io; 1102 struct hinic_qp *qp = &func_to_io->qps[i]; 1103 1104 if (i >= hinic_hwdev_num_qps(hwdev)) 1105 return NULL; 1106 1107 return &qp->rq; 1108 } 1109 1110 /** 1111 * hinic_hwdev_msix_cnt_set - clear message attribute counters for msix entry 1112 * @hwdev: the NIC HW device 1113 * @msix_index: msix_index 1114 * 1115 * Return 0 - Success, negative - Failure 1116 **/ 1117 int hinic_hwdev_msix_cnt_set(struct hinic_hwdev *hwdev, u16 msix_index) 1118 { 1119 return hinic_msix_attr_cnt_clear(hwdev->hwif, msix_index); 1120 } 1121 1122 /** 1123 * hinic_hwdev_msix_set - set message attribute for msix entry 1124 * @hwdev: the NIC HW device 1125 * @msix_index: msix_index 1126 * @pending_limit: the maximum pending interrupt events (unit 8) 1127 * @coalesc_timer: coalesc period for interrupt (unit 8 us) 1128 * @lli_timer_cfg: replenishing period for low latency credit (unit 8 us) 1129 * @lli_credit_limit: maximum credits for low latency msix messages (unit 8) 1130 * @resend_timer: maximum wait for resending msix (unit coalesc period) 1131 * 1132 * Return 0 - Success, negative - Failure 1133 **/ 1134 int hinic_hwdev_msix_set(struct hinic_hwdev *hwdev, u16 msix_index, 1135 u8 pending_limit, u8 coalesc_timer, 1136 u8 lli_timer_cfg, u8 lli_credit_limit, 1137 u8 resend_timer) 1138 { 1139 return hinic_msix_attr_set(hwdev->hwif, msix_index, 1140 pending_limit, coalesc_timer, 1141 lli_timer_cfg, lli_credit_limit, 1142 resend_timer); 1143 } 1144 1145 /** 1146 * hinic_hwdev_hw_ci_addr_set - set cons idx addr and attributes in HW for sq 1147 * @hwdev: the NIC HW device 1148 * @sq: send queue 1149 * @pending_limit: the maximum pending update ci events (unit 8) 1150 * @coalesc_timer: coalesc period for update ci (unit 8 us) 1151 * 1152 * Return 0 - Success, negative - Failure 1153 **/ 1154 int hinic_hwdev_hw_ci_addr_set(struct hinic_hwdev *hwdev, struct hinic_sq *sq, 1155 u8 pending_limit, u8 coalesc_timer) 1156 { 1157 struct hinic_qp *qp = container_of(sq, struct hinic_qp, sq); 1158 struct hinic_hwif *hwif = hwdev->hwif; 1159 struct hinic_pfhwdev *pfhwdev; 1160 struct hinic_cmd_hw_ci hw_ci; 1161 1162 hw_ci.dma_attr_off = 0; 1163 hw_ci.pending_limit = pending_limit; 1164 hw_ci.coalesc_timer = coalesc_timer; 1165 1166 hw_ci.msix_en = 1; 1167 hw_ci.msix_entry_idx = sq->msix_entry; 1168 1169 hw_ci.func_idx = HINIC_HWIF_FUNC_IDX(hwif); 1170 1171 hw_ci.sq_id = qp->q_id; 1172 1173 hw_ci.ci_addr = ADDR_IN_4BYTES(sq->hw_ci_dma_addr); 1174 1175 pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); 1176 return hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, 1177 HINIC_MOD_COMM, 1178 HINIC_COMM_CMD_SQ_HI_CI_SET, 1179 &hw_ci, sizeof(hw_ci), NULL, 1180 NULL, HINIC_MGMT_MSG_SYNC); 1181 } 1182 1183 /** 1184 * hinic_hwdev_set_msix_state- set msix state 1185 * @hwdev: the NIC HW device 1186 * @msix_index: IRQ corresponding index number 1187 * @flag: msix state 1188 * 1189 **/ 1190 void hinic_hwdev_set_msix_state(struct hinic_hwdev *hwdev, u16 msix_index, 1191 enum hinic_msix_state flag) 1192 { 1193 hinic_set_msix_state(hwdev->hwif, msix_index, flag); 1194 } 1195 1196 int hinic_get_board_info(struct hinic_hwdev *hwdev, 1197 struct hinic_comm_board_info *board_info) 1198 { 1199 u16 out_size = sizeof(*board_info); 1200 struct hinic_pfhwdev *pfhwdev; 1201 int err; 1202 1203 if (!hwdev || !board_info) 1204 return -EINVAL; 1205 1206 pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); 1207 1208 err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM, 1209 HINIC_COMM_CMD_GET_BOARD_INFO, 1210 board_info, sizeof(*board_info), 1211 board_info, &out_size, HINIC_MGMT_MSG_SYNC); 1212 if (err || board_info->status || !out_size) { 1213 dev_err(&hwdev->hwif->pdev->dev, 1214 "Failed to get board info, err: %d, status: 0x%x, out size: 0x%x\n", 1215 err, board_info->status, out_size); 1216 return -EIO; 1217 } 1218 1219 return 0; 1220 } 1221