1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ 3 4 #include <linux/kernel.h> 5 #include <linux/types.h> 6 #include <linux/errno.h> 7 #include <linux/io.h> 8 #include <linux/slab.h> 9 #include <linux/etherdevice.h> 10 #include "ionic.h" 11 #include "ionic_dev.h" 12 #include "ionic_lif.h" 13 14 static void ionic_watchdog_cb(struct timer_list *t) 15 { 16 struct ionic *ionic = from_timer(ionic, t, watchdog_timer); 17 struct ionic_lif *lif = ionic->lif; 18 struct ionic_deferred_work *work; 19 int hb; 20 21 mod_timer(&ionic->watchdog_timer, 22 round_jiffies(jiffies + ionic->watchdog_period)); 23 24 if (!lif) 25 return; 26 27 hb = ionic_heartbeat_check(ionic); 28 dev_dbg(ionic->dev, "%s: hb %d running %d UP %d\n", 29 __func__, hb, netif_running(lif->netdev), 30 test_bit(IONIC_LIF_F_UP, lif->state)); 31 32 if (hb >= 0 && 33 !test_bit(IONIC_LIF_F_FW_RESET, lif->state)) 34 ionic_link_status_check_request(lif, CAN_NOT_SLEEP); 35 36 if (test_bit(IONIC_LIF_F_FILTER_SYNC_NEEDED, lif->state) && 37 !test_bit(IONIC_LIF_F_FW_RESET, lif->state)) { 38 work = kzalloc(sizeof(*work), GFP_ATOMIC); 39 if (!work) { 40 netdev_err(lif->netdev, "rxmode change dropped\n"); 41 return; 42 } 43 44 work->type = IONIC_DW_TYPE_RX_MODE; 45 netdev_dbg(lif->netdev, "deferred: rx_mode\n"); 46 ionic_lif_deferred_enqueue(&lif->deferred, work); 47 } 48 } 49 50 static void ionic_watchdog_init(struct ionic *ionic) 51 { 52 struct ionic_dev *idev = &ionic->idev; 53 54 timer_setup(&ionic->watchdog_timer, ionic_watchdog_cb, 0); 55 ionic->watchdog_period = IONIC_WATCHDOG_SECS * HZ; 56 57 /* set times to ensure the first check will proceed */ 58 atomic_long_set(&idev->last_check_time, jiffies - 2 * HZ); 59 idev->last_hb_time = jiffies - 2 * ionic->watchdog_period; 60 /* init as ready, so no transition if the first check succeeds */ 61 idev->last_fw_hb = 0; 62 idev->fw_hb_ready = true; 63 idev->fw_status_ready = true; 64 idev->fw_generation = IONIC_FW_STS_F_GENERATION & 65 ioread8(&idev->dev_info_regs->fw_status); 66 } 67 68 void ionic_init_devinfo(struct ionic *ionic) 69 { 70 struct ionic_dev *idev = &ionic->idev; 71 72 idev->dev_info.asic_type = ioread8(&idev->dev_info_regs->asic_type); 73 idev->dev_info.asic_rev = ioread8(&idev->dev_info_regs->asic_rev); 74 75 memcpy_fromio(idev->dev_info.fw_version, 76 idev->dev_info_regs->fw_version, 77 IONIC_DEVINFO_FWVERS_BUFLEN); 78 79 memcpy_fromio(idev->dev_info.serial_num, 80 idev->dev_info_regs->serial_num, 81 IONIC_DEVINFO_SERIAL_BUFLEN); 82 83 idev->dev_info.fw_version[IONIC_DEVINFO_FWVERS_BUFLEN] = 0; 84 idev->dev_info.serial_num[IONIC_DEVINFO_SERIAL_BUFLEN] = 0; 85 86 dev_dbg(ionic->dev, "fw_version %s\n", idev->dev_info.fw_version); 87 } 88 89 int ionic_dev_setup(struct ionic *ionic) 90 { 91 struct ionic_dev_bar *bar = ionic->bars; 92 unsigned int num_bars = ionic->num_bars; 93 struct ionic_dev *idev = &ionic->idev; 94 struct device *dev = ionic->dev; 95 int size; 96 u32 sig; 97 98 /* BAR0: dev_cmd and interrupts */ 99 if (num_bars < 1) { 100 dev_err(dev, "No bars found, aborting\n"); 101 return -EFAULT; 102 } 103 104 if (bar->len < IONIC_BAR0_SIZE) { 105 dev_err(dev, "Resource bar size %lu too small, aborting\n", 106 bar->len); 107 return -EFAULT; 108 } 109 110 idev->dev_info_regs = bar->vaddr + IONIC_BAR0_DEV_INFO_REGS_OFFSET; 111 idev->dev_cmd_regs = bar->vaddr + IONIC_BAR0_DEV_CMD_REGS_OFFSET; 112 idev->intr_status = bar->vaddr + IONIC_BAR0_INTR_STATUS_OFFSET; 113 idev->intr_ctrl = bar->vaddr + IONIC_BAR0_INTR_CTRL_OFFSET; 114 115 idev->hwstamp_regs = &idev->dev_info_regs->hwstamp; 116 117 sig = ioread32(&idev->dev_info_regs->signature); 118 if (sig != IONIC_DEV_INFO_SIGNATURE) { 119 dev_err(dev, "Incompatible firmware signature %x", sig); 120 return -EFAULT; 121 } 122 123 ionic_init_devinfo(ionic); 124 125 /* BAR1: doorbells */ 126 bar++; 127 if (num_bars < 2) { 128 dev_err(dev, "Doorbell bar missing, aborting\n"); 129 return -EFAULT; 130 } 131 132 ionic_watchdog_init(ionic); 133 134 idev->db_pages = bar->vaddr; 135 idev->phy_db_pages = bar->bus_addr; 136 137 /* BAR2: optional controller memory mapping */ 138 bar++; 139 mutex_init(&idev->cmb_inuse_lock); 140 if (num_bars < 3 || !ionic->bars[IONIC_PCI_BAR_CMB].len) { 141 idev->cmb_inuse = NULL; 142 return 0; 143 } 144 145 idev->phy_cmb_pages = bar->bus_addr; 146 idev->cmb_npages = bar->len / PAGE_SIZE; 147 size = BITS_TO_LONGS(idev->cmb_npages) * sizeof(long); 148 idev->cmb_inuse = kzalloc(size, GFP_KERNEL); 149 if (!idev->cmb_inuse) 150 dev_warn(dev, "No memory for CMB, disabling\n"); 151 152 return 0; 153 } 154 155 void ionic_dev_teardown(struct ionic *ionic) 156 { 157 struct ionic_dev *idev = &ionic->idev; 158 159 kfree(idev->cmb_inuse); 160 idev->cmb_inuse = NULL; 161 idev->phy_cmb_pages = 0; 162 idev->cmb_npages = 0; 163 164 mutex_destroy(&idev->cmb_inuse_lock); 165 } 166 167 /* Devcmd Interface */ 168 static bool __ionic_is_fw_running(struct ionic_dev *idev, u8 *status_ptr) 169 { 170 u8 fw_status; 171 172 if (!idev->dev_info_regs) { 173 if (status_ptr) 174 *status_ptr = 0xff; 175 return false; 176 } 177 178 fw_status = ioread8(&idev->dev_info_regs->fw_status); 179 if (status_ptr) 180 *status_ptr = fw_status; 181 182 /* firmware is useful only if the running bit is set and 183 * fw_status != 0xff (bad PCI read) 184 */ 185 return (fw_status != 0xff) && (fw_status & IONIC_FW_STS_F_RUNNING); 186 } 187 188 bool ionic_is_fw_running(struct ionic_dev *idev) 189 { 190 return __ionic_is_fw_running(idev, NULL); 191 } 192 193 int ionic_heartbeat_check(struct ionic *ionic) 194 { 195 unsigned long check_time, last_check_time; 196 struct ionic_dev *idev = &ionic->idev; 197 struct ionic_lif *lif = ionic->lif; 198 bool fw_status_ready = true; 199 bool fw_hb_ready; 200 u8 fw_generation; 201 u8 fw_status; 202 u32 fw_hb; 203 204 /* wait a least one second before testing again */ 205 check_time = jiffies; 206 last_check_time = atomic_long_read(&idev->last_check_time); 207 do_check_time: 208 if (time_before(check_time, last_check_time + HZ)) 209 return 0; 210 if (!atomic_long_try_cmpxchg_relaxed(&idev->last_check_time, 211 &last_check_time, check_time)) { 212 /* if called concurrently, only the first should proceed. */ 213 dev_dbg(ionic->dev, "%s: do_check_time again\n", __func__); 214 goto do_check_time; 215 } 216 217 /* If fw_status is not ready don't bother with the generation */ 218 if (!__ionic_is_fw_running(idev, &fw_status)) { 219 fw_status_ready = false; 220 } else { 221 fw_generation = fw_status & IONIC_FW_STS_F_GENERATION; 222 if (idev->fw_generation != fw_generation) { 223 dev_info(ionic->dev, "FW generation 0x%02x -> 0x%02x\n", 224 idev->fw_generation, fw_generation); 225 226 idev->fw_generation = fw_generation; 227 228 /* If the generation changed, the fw status is not 229 * ready so we need to trigger a fw-down cycle. After 230 * the down, the next watchdog will see the fw is up 231 * and the generation value stable, so will trigger 232 * the fw-up activity. 233 * 234 * If we had already moved to FW_RESET from a RESET event, 235 * it is possible that we never saw the fw_status go to 0, 236 * so we fake the current idev->fw_status_ready here to 237 * force the transition and get FW up again. 238 */ 239 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state)) 240 idev->fw_status_ready = false; /* go to running */ 241 else 242 fw_status_ready = false; /* go to down */ 243 } 244 } 245 246 dev_dbg(ionic->dev, "fw_status 0x%02x ready %d idev->ready %d last_hb 0x%x state 0x%02lx\n", 247 fw_status, fw_status_ready, idev->fw_status_ready, 248 idev->last_fw_hb, lif->state[0]); 249 250 /* is this a transition? */ 251 if (fw_status_ready != idev->fw_status_ready && 252 !test_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) { 253 bool trigger = false; 254 255 idev->fw_status_ready = fw_status_ready; 256 257 if (!fw_status_ready && 258 !test_bit(IONIC_LIF_F_FW_RESET, lif->state) && 259 !test_and_set_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) { 260 dev_info(ionic->dev, "FW stopped 0x%02x\n", fw_status); 261 trigger = true; 262 263 } else if (fw_status_ready && 264 test_bit(IONIC_LIF_F_FW_RESET, lif->state)) { 265 dev_info(ionic->dev, "FW running 0x%02x\n", fw_status); 266 trigger = true; 267 } 268 269 if (trigger) { 270 struct ionic_deferred_work *work; 271 272 work = kzalloc(sizeof(*work), GFP_ATOMIC); 273 if (work) { 274 work->type = IONIC_DW_TYPE_LIF_RESET; 275 work->fw_status = fw_status_ready; 276 ionic_lif_deferred_enqueue(&lif->deferred, work); 277 } 278 } 279 } 280 281 if (!idev->fw_status_ready) 282 return -ENXIO; 283 284 /* Because of some variability in the actual FW heartbeat, we 285 * wait longer than the DEVCMD_TIMEOUT before checking again. 286 */ 287 last_check_time = idev->last_hb_time; 288 if (time_before(check_time, last_check_time + DEVCMD_TIMEOUT * 2 * HZ)) 289 return 0; 290 291 fw_hb = ioread32(&idev->dev_info_regs->fw_heartbeat); 292 fw_hb_ready = fw_hb != idev->last_fw_hb; 293 294 /* early FW version had no heartbeat, so fake it */ 295 if (!fw_hb_ready && !fw_hb) 296 fw_hb_ready = true; 297 298 dev_dbg(ionic->dev, "%s: fw_hb %u last_fw_hb %u ready %u\n", 299 __func__, fw_hb, idev->last_fw_hb, fw_hb_ready); 300 301 idev->last_fw_hb = fw_hb; 302 303 /* log a transition */ 304 if (fw_hb_ready != idev->fw_hb_ready) { 305 idev->fw_hb_ready = fw_hb_ready; 306 if (!fw_hb_ready) 307 dev_info(ionic->dev, "FW heartbeat stalled at %d\n", fw_hb); 308 else 309 dev_info(ionic->dev, "FW heartbeat restored at %d\n", fw_hb); 310 } 311 312 if (!fw_hb_ready) 313 return -ENXIO; 314 315 idev->last_hb_time = check_time; 316 317 return 0; 318 } 319 320 u8 ionic_dev_cmd_status(struct ionic_dev *idev) 321 { 322 if (!idev->dev_cmd_regs) 323 return (u8)PCI_ERROR_RESPONSE; 324 return ioread8(&idev->dev_cmd_regs->comp.comp.status); 325 } 326 327 bool ionic_dev_cmd_done(struct ionic_dev *idev) 328 { 329 if (!idev->dev_cmd_regs) 330 return false; 331 return ioread32(&idev->dev_cmd_regs->done) & IONIC_DEV_CMD_DONE; 332 } 333 334 void ionic_dev_cmd_comp(struct ionic_dev *idev, union ionic_dev_cmd_comp *comp) 335 { 336 if (!idev->dev_cmd_regs) 337 return; 338 memcpy_fromio(comp, &idev->dev_cmd_regs->comp, sizeof(*comp)); 339 } 340 341 void ionic_dev_cmd_go(struct ionic_dev *idev, union ionic_dev_cmd *cmd) 342 { 343 idev->opcode = cmd->cmd.opcode; 344 345 if (!idev->dev_cmd_regs) 346 return; 347 348 memcpy_toio(&idev->dev_cmd_regs->cmd, cmd, sizeof(*cmd)); 349 iowrite32(0, &idev->dev_cmd_regs->done); 350 iowrite32(1, &idev->dev_cmd_regs->doorbell); 351 } 352 353 /* Device commands */ 354 void ionic_dev_cmd_identify(struct ionic_dev *idev, u8 ver) 355 { 356 union ionic_dev_cmd cmd = { 357 .identify.opcode = IONIC_CMD_IDENTIFY, 358 .identify.ver = ver, 359 }; 360 361 ionic_dev_cmd_go(idev, &cmd); 362 } 363 364 void ionic_dev_cmd_init(struct ionic_dev *idev) 365 { 366 union ionic_dev_cmd cmd = { 367 .init.opcode = IONIC_CMD_INIT, 368 .init.type = 0, 369 }; 370 371 ionic_dev_cmd_go(idev, &cmd); 372 } 373 374 void ionic_dev_cmd_reset(struct ionic_dev *idev) 375 { 376 union ionic_dev_cmd cmd = { 377 .reset.opcode = IONIC_CMD_RESET, 378 }; 379 380 ionic_dev_cmd_go(idev, &cmd); 381 } 382 383 /* Port commands */ 384 void ionic_dev_cmd_port_identify(struct ionic_dev *idev) 385 { 386 union ionic_dev_cmd cmd = { 387 .port_init.opcode = IONIC_CMD_PORT_IDENTIFY, 388 .port_init.index = 0, 389 }; 390 391 ionic_dev_cmd_go(idev, &cmd); 392 } 393 394 void ionic_dev_cmd_port_init(struct ionic_dev *idev) 395 { 396 union ionic_dev_cmd cmd = { 397 .port_init.opcode = IONIC_CMD_PORT_INIT, 398 .port_init.index = 0, 399 .port_init.info_pa = cpu_to_le64(idev->port_info_pa), 400 }; 401 402 ionic_dev_cmd_go(idev, &cmd); 403 } 404 405 void ionic_dev_cmd_port_reset(struct ionic_dev *idev) 406 { 407 union ionic_dev_cmd cmd = { 408 .port_reset.opcode = IONIC_CMD_PORT_RESET, 409 .port_reset.index = 0, 410 }; 411 412 ionic_dev_cmd_go(idev, &cmd); 413 } 414 415 void ionic_dev_cmd_port_state(struct ionic_dev *idev, u8 state) 416 { 417 union ionic_dev_cmd cmd = { 418 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, 419 .port_setattr.index = 0, 420 .port_setattr.attr = IONIC_PORT_ATTR_STATE, 421 .port_setattr.state = state, 422 }; 423 424 ionic_dev_cmd_go(idev, &cmd); 425 } 426 427 void ionic_dev_cmd_port_speed(struct ionic_dev *idev, u32 speed) 428 { 429 union ionic_dev_cmd cmd = { 430 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, 431 .port_setattr.index = 0, 432 .port_setattr.attr = IONIC_PORT_ATTR_SPEED, 433 .port_setattr.speed = cpu_to_le32(speed), 434 }; 435 436 ionic_dev_cmd_go(idev, &cmd); 437 } 438 439 void ionic_dev_cmd_port_autoneg(struct ionic_dev *idev, u8 an_enable) 440 { 441 union ionic_dev_cmd cmd = { 442 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, 443 .port_setattr.index = 0, 444 .port_setattr.attr = IONIC_PORT_ATTR_AUTONEG, 445 .port_setattr.an_enable = an_enable, 446 }; 447 448 ionic_dev_cmd_go(idev, &cmd); 449 } 450 451 void ionic_dev_cmd_port_fec(struct ionic_dev *idev, u8 fec_type) 452 { 453 union ionic_dev_cmd cmd = { 454 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, 455 .port_setattr.index = 0, 456 .port_setattr.attr = IONIC_PORT_ATTR_FEC, 457 .port_setattr.fec_type = fec_type, 458 }; 459 460 ionic_dev_cmd_go(idev, &cmd); 461 } 462 463 void ionic_dev_cmd_port_pause(struct ionic_dev *idev, u8 pause_type) 464 { 465 union ionic_dev_cmd cmd = { 466 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, 467 .port_setattr.index = 0, 468 .port_setattr.attr = IONIC_PORT_ATTR_PAUSE, 469 .port_setattr.pause_type = pause_type, 470 }; 471 472 ionic_dev_cmd_go(idev, &cmd); 473 } 474 475 /* VF commands */ 476 int ionic_set_vf_config(struct ionic *ionic, int vf, 477 struct ionic_vf_setattr_cmd *vfc) 478 { 479 union ionic_dev_cmd cmd = { 480 .vf_setattr.opcode = IONIC_CMD_VF_SETATTR, 481 .vf_setattr.attr = vfc->attr, 482 .vf_setattr.vf_index = cpu_to_le16(vf), 483 }; 484 int err; 485 486 memcpy(cmd.vf_setattr.pad, vfc->pad, sizeof(vfc->pad)); 487 488 mutex_lock(&ionic->dev_cmd_lock); 489 ionic_dev_cmd_go(&ionic->idev, &cmd); 490 err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT); 491 mutex_unlock(&ionic->dev_cmd_lock); 492 493 return err; 494 } 495 496 int ionic_dev_cmd_vf_getattr(struct ionic *ionic, int vf, u8 attr, 497 struct ionic_vf_getattr_comp *comp) 498 { 499 union ionic_dev_cmd cmd = { 500 .vf_getattr.opcode = IONIC_CMD_VF_GETATTR, 501 .vf_getattr.attr = attr, 502 .vf_getattr.vf_index = cpu_to_le16(vf), 503 }; 504 int err; 505 506 if (vf >= ionic->num_vfs) 507 return -EINVAL; 508 509 switch (attr) { 510 case IONIC_VF_ATTR_SPOOFCHK: 511 case IONIC_VF_ATTR_TRUST: 512 case IONIC_VF_ATTR_LINKSTATE: 513 case IONIC_VF_ATTR_MAC: 514 case IONIC_VF_ATTR_VLAN: 515 case IONIC_VF_ATTR_RATE: 516 break; 517 case IONIC_VF_ATTR_STATSADDR: 518 default: 519 return -EINVAL; 520 } 521 522 mutex_lock(&ionic->dev_cmd_lock); 523 ionic_dev_cmd_go(&ionic->idev, &cmd); 524 err = ionic_dev_cmd_wait_nomsg(ionic, DEVCMD_TIMEOUT); 525 memcpy_fromio(comp, &ionic->idev.dev_cmd_regs->comp.vf_getattr, 526 sizeof(*comp)); 527 mutex_unlock(&ionic->dev_cmd_lock); 528 529 if (err && comp->status != IONIC_RC_ENOSUPP) 530 ionic_dev_cmd_dev_err_print(ionic, cmd.vf_getattr.opcode, 531 comp->status, err); 532 533 return err; 534 } 535 536 void ionic_vf_start(struct ionic *ionic) 537 { 538 union ionic_dev_cmd cmd = { 539 .vf_ctrl.opcode = IONIC_CMD_VF_CTRL, 540 .vf_ctrl.ctrl_opcode = IONIC_VF_CTRL_START_ALL, 541 }; 542 543 if (!(ionic->ident.dev.capabilities & cpu_to_le64(IONIC_DEV_CAP_VF_CTRL))) 544 return; 545 546 ionic_dev_cmd_go(&ionic->idev, &cmd); 547 ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT); 548 } 549 550 /* LIF commands */ 551 void ionic_dev_cmd_queue_identify(struct ionic_dev *idev, 552 u16 lif_type, u8 qtype, u8 qver) 553 { 554 union ionic_dev_cmd cmd = { 555 .q_identify.opcode = IONIC_CMD_Q_IDENTIFY, 556 .q_identify.lif_type = cpu_to_le16(lif_type), 557 .q_identify.type = qtype, 558 .q_identify.ver = qver, 559 }; 560 561 ionic_dev_cmd_go(idev, &cmd); 562 } 563 564 void ionic_dev_cmd_lif_identify(struct ionic_dev *idev, u8 type, u8 ver) 565 { 566 union ionic_dev_cmd cmd = { 567 .lif_identify.opcode = IONIC_CMD_LIF_IDENTIFY, 568 .lif_identify.type = type, 569 .lif_identify.ver = ver, 570 }; 571 572 ionic_dev_cmd_go(idev, &cmd); 573 } 574 575 void ionic_dev_cmd_lif_init(struct ionic_dev *idev, u16 lif_index, 576 dma_addr_t info_pa) 577 { 578 union ionic_dev_cmd cmd = { 579 .lif_init.opcode = IONIC_CMD_LIF_INIT, 580 .lif_init.index = cpu_to_le16(lif_index), 581 .lif_init.info_pa = cpu_to_le64(info_pa), 582 }; 583 584 ionic_dev_cmd_go(idev, &cmd); 585 } 586 587 void ionic_dev_cmd_lif_reset(struct ionic_dev *idev, u16 lif_index) 588 { 589 union ionic_dev_cmd cmd = { 590 .lif_init.opcode = IONIC_CMD_LIF_RESET, 591 .lif_init.index = cpu_to_le16(lif_index), 592 }; 593 594 ionic_dev_cmd_go(idev, &cmd); 595 } 596 597 void ionic_dev_cmd_adminq_init(struct ionic_dev *idev, struct ionic_qcq *qcq, 598 u16 lif_index, u16 intr_index) 599 { 600 struct ionic_queue *q = &qcq->q; 601 struct ionic_cq *cq = &qcq->cq; 602 603 union ionic_dev_cmd cmd = { 604 .q_init.opcode = IONIC_CMD_Q_INIT, 605 .q_init.lif_index = cpu_to_le16(lif_index), 606 .q_init.type = q->type, 607 .q_init.ver = qcq->q.lif->qtype_info[q->type].version, 608 .q_init.index = cpu_to_le32(q->index), 609 .q_init.flags = cpu_to_le16(IONIC_QINIT_F_IRQ | 610 IONIC_QINIT_F_ENA), 611 .q_init.pid = cpu_to_le16(q->pid), 612 .q_init.intr_index = cpu_to_le16(intr_index), 613 .q_init.ring_size = ilog2(q->num_descs), 614 .q_init.ring_base = cpu_to_le64(q->base_pa), 615 .q_init.cq_ring_base = cpu_to_le64(cq->base_pa), 616 }; 617 618 ionic_dev_cmd_go(idev, &cmd); 619 } 620 621 int ionic_db_page_num(struct ionic_lif *lif, int pid) 622 { 623 return (lif->hw_index * lif->dbid_count) + pid; 624 } 625 626 int ionic_get_cmb(struct ionic_lif *lif, u32 *pgid, phys_addr_t *pgaddr, int order) 627 { 628 struct ionic_dev *idev = &lif->ionic->idev; 629 int ret; 630 631 mutex_lock(&idev->cmb_inuse_lock); 632 ret = bitmap_find_free_region(idev->cmb_inuse, idev->cmb_npages, order); 633 mutex_unlock(&idev->cmb_inuse_lock); 634 635 if (ret < 0) 636 return ret; 637 638 *pgid = ret; 639 *pgaddr = idev->phy_cmb_pages + ret * PAGE_SIZE; 640 641 return 0; 642 } 643 644 void ionic_put_cmb(struct ionic_lif *lif, u32 pgid, int order) 645 { 646 struct ionic_dev *idev = &lif->ionic->idev; 647 648 mutex_lock(&idev->cmb_inuse_lock); 649 bitmap_release_region(idev->cmb_inuse, pgid, order); 650 mutex_unlock(&idev->cmb_inuse_lock); 651 } 652 653 int ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq, 654 struct ionic_intr_info *intr, 655 unsigned int num_descs, size_t desc_size) 656 { 657 unsigned int ring_size; 658 659 if (desc_size == 0 || !is_power_of_2(num_descs)) 660 return -EINVAL; 661 662 ring_size = ilog2(num_descs); 663 if (ring_size < 2 || ring_size > 16) 664 return -EINVAL; 665 666 cq->lif = lif; 667 cq->bound_intr = intr; 668 cq->num_descs = num_descs; 669 cq->desc_size = desc_size; 670 cq->tail_idx = 0; 671 cq->done_color = 1; 672 673 return 0; 674 } 675 676 void ionic_cq_map(struct ionic_cq *cq, void *base, dma_addr_t base_pa) 677 { 678 struct ionic_cq_info *cur; 679 unsigned int i; 680 681 cq->base = base; 682 cq->base_pa = base_pa; 683 684 for (i = 0, cur = cq->info; i < cq->num_descs; i++, cur++) 685 cur->cq_desc = base + (i * cq->desc_size); 686 } 687 688 void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q) 689 { 690 cq->bound_q = q; 691 } 692 693 unsigned int ionic_cq_service(struct ionic_cq *cq, unsigned int work_to_do, 694 ionic_cq_cb cb, ionic_cq_done_cb done_cb, 695 void *done_arg) 696 { 697 struct ionic_cq_info *cq_info; 698 unsigned int work_done = 0; 699 700 if (work_to_do == 0) 701 return 0; 702 703 cq_info = &cq->info[cq->tail_idx]; 704 while (cb(cq, cq_info)) { 705 if (cq->tail_idx == cq->num_descs - 1) 706 cq->done_color = !cq->done_color; 707 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); 708 cq_info = &cq->info[cq->tail_idx]; 709 710 if (++work_done >= work_to_do) 711 break; 712 } 713 714 if (work_done && done_cb) 715 done_cb(done_arg); 716 717 return work_done; 718 } 719 720 int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev, 721 struct ionic_queue *q, unsigned int index, const char *name, 722 unsigned int num_descs, size_t desc_size, 723 size_t sg_desc_size, unsigned int pid) 724 { 725 unsigned int ring_size; 726 727 if (desc_size == 0 || !is_power_of_2(num_descs)) 728 return -EINVAL; 729 730 ring_size = ilog2(num_descs); 731 if (ring_size < 2 || ring_size > 16) 732 return -EINVAL; 733 734 q->lif = lif; 735 q->idev = idev; 736 q->index = index; 737 q->num_descs = num_descs; 738 q->desc_size = desc_size; 739 q->sg_desc_size = sg_desc_size; 740 q->tail_idx = 0; 741 q->head_idx = 0; 742 q->pid = pid; 743 744 snprintf(q->name, sizeof(q->name), "L%d-%s%u", lif->index, name, index); 745 746 return 0; 747 } 748 749 void ionic_q_map(struct ionic_queue *q, void *base, dma_addr_t base_pa) 750 { 751 struct ionic_desc_info *cur; 752 unsigned int i; 753 754 q->base = base; 755 q->base_pa = base_pa; 756 757 for (i = 0, cur = q->info; i < q->num_descs; i++, cur++) 758 cur->desc = base + (i * q->desc_size); 759 } 760 761 void ionic_q_cmb_map(struct ionic_queue *q, void __iomem *base, dma_addr_t base_pa) 762 { 763 struct ionic_desc_info *cur; 764 unsigned int i; 765 766 q->cmb_base = base; 767 q->cmb_base_pa = base_pa; 768 769 for (i = 0, cur = q->info; i < q->num_descs; i++, cur++) 770 cur->cmb_desc = base + (i * q->desc_size); 771 } 772 773 void ionic_q_sg_map(struct ionic_queue *q, void *base, dma_addr_t base_pa) 774 { 775 struct ionic_desc_info *cur; 776 unsigned int i; 777 778 q->sg_base = base; 779 q->sg_base_pa = base_pa; 780 781 for (i = 0, cur = q->info; i < q->num_descs; i++, cur++) 782 cur->sg_desc = base + (i * q->sg_desc_size); 783 } 784 785 void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, ionic_desc_cb cb, 786 void *cb_arg) 787 { 788 struct ionic_desc_info *desc_info; 789 struct ionic_lif *lif = q->lif; 790 struct device *dev = q->dev; 791 792 desc_info = &q->info[q->head_idx]; 793 desc_info->cb = cb; 794 desc_info->cb_arg = cb_arg; 795 796 q->head_idx = (q->head_idx + 1) & (q->num_descs - 1); 797 798 dev_dbg(dev, "lif=%d qname=%s qid=%d qtype=%d p_index=%d ringdb=%d\n", 799 q->lif->index, q->name, q->hw_type, q->hw_index, 800 q->head_idx, ring_doorbell); 801 802 if (ring_doorbell) { 803 ionic_dbell_ring(lif->kern_dbpage, q->hw_type, 804 q->dbval | q->head_idx); 805 806 q->dbell_jiffies = jiffies; 807 808 if (q_to_qcq(q)->napi_qcq) 809 mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline, 810 jiffies + IONIC_NAPI_DEADLINE); 811 } 812 } 813 814 static bool ionic_q_is_posted(struct ionic_queue *q, unsigned int pos) 815 { 816 unsigned int mask, tail, head; 817 818 mask = q->num_descs - 1; 819 tail = q->tail_idx; 820 head = q->head_idx; 821 822 return ((pos - tail) & mask) < ((head - tail) & mask); 823 } 824 825 void ionic_q_service(struct ionic_queue *q, struct ionic_cq_info *cq_info, 826 unsigned int stop_index) 827 { 828 struct ionic_desc_info *desc_info; 829 ionic_desc_cb cb; 830 void *cb_arg; 831 u16 index; 832 833 /* check for empty queue */ 834 if (q->tail_idx == q->head_idx) 835 return; 836 837 /* stop index must be for a descriptor that is not yet completed */ 838 if (unlikely(!ionic_q_is_posted(q, stop_index))) 839 dev_err(q->dev, 840 "ionic stop is not posted %s stop %u tail %u head %u\n", 841 q->name, stop_index, q->tail_idx, q->head_idx); 842 843 do { 844 desc_info = &q->info[q->tail_idx]; 845 index = q->tail_idx; 846 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 847 848 cb = desc_info->cb; 849 cb_arg = desc_info->cb_arg; 850 851 desc_info->cb = NULL; 852 desc_info->cb_arg = NULL; 853 854 if (cb) 855 cb(q, desc_info, cq_info, cb_arg); 856 } while (index != stop_index); 857 } 858