1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * SolidRun DPU driver for control plane 4 * 5 * Copyright (C) 2022-2023 SolidRun 6 * 7 * Author: Alvaro Karsz <alvaro.karsz@solid-run.com> 8 * 9 */ 10 #include <linux/iopoll.h> 11 12 #include "snet_vdpa.h" 13 14 /* SNET DPU device ID */ 15 #define SNET_DEVICE_ID 0x1000 16 /* SNET signature */ 17 #define SNET_SIGNATURE 0xD0D06363 18 /* Max. config version that we can work with */ 19 #define SNET_CFG_VERSION 0x2 20 /* Queue align */ 21 #define SNET_QUEUE_ALIGNMENT PAGE_SIZE 22 /* Kick value to notify that new data is available */ 23 #define SNET_KICK_VAL 0x1 24 #define SNET_CONFIG_OFF 0x0 25 /* How long we are willing to wait for a SNET device */ 26 #define SNET_DETECT_TIMEOUT 5000000 27 /* How long should we wait for the DPU to read our config */ 28 #define SNET_READ_CFG_TIMEOUT 3000000 29 /* Size of configs written to the DPU */ 30 #define SNET_GENERAL_CFG_LEN 36 31 #define SNET_GENERAL_CFG_VQ_LEN 40 32 33 static struct snet *vdpa_to_snet(struct vdpa_device *vdpa) 34 { 35 return container_of(vdpa, struct snet, vdpa); 36 } 37 38 static irqreturn_t snet_cfg_irq_hndlr(int irq, void *data) 39 { 40 struct snet *snet = data; 41 /* Call callback if any */ 42 if (likely(snet->cb.callback)) 43 return snet->cb.callback(snet->cb.private); 44 45 return IRQ_HANDLED; 46 } 47 48 static irqreturn_t snet_vq_irq_hndlr(int irq, void *data) 49 { 50 struct snet_vq *vq = data; 51 /* Call callback if any */ 52 if (likely(vq->cb.callback)) 53 return vq->cb.callback(vq->cb.private); 54 55 return IRQ_HANDLED; 56 } 57 58 static void snet_free_irqs(struct snet *snet) 59 { 60 struct psnet *psnet = snet->psnet; 61 struct pci_dev *pdev; 62 u32 i; 63 64 /* Which Device allcoated the IRQs? */ 65 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF)) 66 pdev = snet->pdev->physfn; 67 else 68 pdev = snet->pdev; 69 70 /* Free config's IRQ */ 71 if (snet->cfg_irq != -1) { 72 devm_free_irq(&pdev->dev, snet->cfg_irq, snet); 73 snet->cfg_irq = -1; 74 } 75 /* Free VQ IRQs */ 76 for (i = 0; i < snet->cfg->vq_num; i++) { 77 if (snet->vqs[i] && snet->vqs[i]->irq != -1) { 78 devm_free_irq(&pdev->dev, snet->vqs[i]->irq, snet->vqs[i]); 79 snet->vqs[i]->irq = -1; 80 } 81 } 82 83 /* IRQ vectors are freed when the pci remove callback is called */ 84 } 85 86 static int snet_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area, 87 u64 driver_area, u64 device_area) 88 { 89 struct snet *snet = vdpa_to_snet(vdev); 90 /* save received parameters in vqueue sturct */ 91 snet->vqs[idx]->desc_area = desc_area; 92 snet->vqs[idx]->driver_area = driver_area; 93 snet->vqs[idx]->device_area = device_area; 94 95 return 0; 96 } 97 98 static void snet_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) 99 { 100 struct snet *snet = vdpa_to_snet(vdev); 101 /* save num in vqueue */ 102 snet->vqs[idx]->num = num; 103 } 104 105 static void snet_kick_vq(struct vdpa_device *vdev, u16 idx) 106 { 107 struct snet *snet = vdpa_to_snet(vdev); 108 /* not ready - ignore */ 109 if (unlikely(!snet->vqs[idx]->ready)) 110 return; 111 112 iowrite32(SNET_KICK_VAL, snet->vqs[idx]->kick_ptr); 113 } 114 115 static void snet_kick_vq_with_data(struct vdpa_device *vdev, u32 data) 116 { 117 struct snet *snet = vdpa_to_snet(vdev); 118 u16 idx = data & 0xFFFF; 119 120 /* not ready - ignore */ 121 if (unlikely(!snet->vqs[idx]->ready)) 122 return; 123 124 iowrite32((data & 0xFFFF0000) | SNET_KICK_VAL, snet->vqs[idx]->kick_ptr); 125 } 126 127 static void snet_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb) 128 { 129 struct snet *snet = vdpa_to_snet(vdev); 130 131 snet->vqs[idx]->cb.callback = cb->callback; 132 snet->vqs[idx]->cb.private = cb->private; 133 } 134 135 static void snet_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready) 136 { 137 struct snet *snet = vdpa_to_snet(vdev); 138 139 snet->vqs[idx]->ready = ready; 140 } 141 142 static bool snet_get_vq_ready(struct vdpa_device *vdev, u16 idx) 143 { 144 struct snet *snet = vdpa_to_snet(vdev); 145 146 return snet->vqs[idx]->ready; 147 } 148 149 static bool snet_vq_state_is_initial(struct snet *snet, const struct vdpa_vq_state *state) 150 { 151 if (SNET_HAS_FEATURE(snet, VIRTIO_F_RING_PACKED)) { 152 const struct vdpa_vq_state_packed *p = &state->packed; 153 154 if (p->last_avail_counter == 1 && p->last_used_counter == 1 && 155 p->last_avail_idx == 0 && p->last_used_idx == 0) 156 return true; 157 } else { 158 const struct vdpa_vq_state_split *s = &state->split; 159 160 if (s->avail_index == 0) 161 return true; 162 } 163 164 return false; 165 } 166 167 static int snet_set_vq_state(struct vdpa_device *vdev, u16 idx, const struct vdpa_vq_state *state) 168 { 169 struct snet *snet = vdpa_to_snet(vdev); 170 171 /* We can set any state for config version 2+ */ 172 if (SNET_CFG_VER(snet, 2)) { 173 memcpy(&snet->vqs[idx]->vq_state, state, sizeof(*state)); 174 return 0; 175 } 176 177 /* Older config - we can't set the VQ state. 178 * Return 0 only if this is the initial state we use in the DPU. 179 */ 180 if (snet_vq_state_is_initial(snet, state)) 181 return 0; 182 183 return -EOPNOTSUPP; 184 } 185 186 static int snet_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state) 187 { 188 struct snet *snet = vdpa_to_snet(vdev); 189 190 return snet_read_vq_state(snet, idx, state); 191 } 192 193 static int snet_get_vq_irq(struct vdpa_device *vdev, u16 idx) 194 { 195 struct snet *snet = vdpa_to_snet(vdev); 196 197 return snet->vqs[idx]->irq; 198 } 199 200 static u32 snet_get_vq_align(struct vdpa_device *vdev) 201 { 202 return (u32)SNET_QUEUE_ALIGNMENT; 203 } 204 205 static int snet_reset_dev(struct snet *snet) 206 { 207 struct pci_dev *pdev = snet->pdev; 208 int ret = 0; 209 u32 i; 210 211 /* If status is 0, nothing to do */ 212 if (!snet->status) 213 return 0; 214 215 /* If DPU started, destroy it */ 216 if (snet->status & VIRTIO_CONFIG_S_DRIVER_OK) 217 ret = snet_destroy_dev(snet); 218 219 /* Clear VQs */ 220 for (i = 0; i < snet->cfg->vq_num; i++) { 221 if (!snet->vqs[i]) 222 continue; 223 snet->vqs[i]->cb.callback = NULL; 224 snet->vqs[i]->cb.private = NULL; 225 snet->vqs[i]->desc_area = 0; 226 snet->vqs[i]->device_area = 0; 227 snet->vqs[i]->driver_area = 0; 228 snet->vqs[i]->ready = false; 229 } 230 231 /* Clear config callback */ 232 snet->cb.callback = NULL; 233 snet->cb.private = NULL; 234 /* Free IRQs */ 235 snet_free_irqs(snet); 236 /* Reset status */ 237 snet->status = 0; 238 snet->dpu_ready = false; 239 240 if (ret) 241 SNET_WARN(pdev, "Incomplete reset to SNET[%u] device, err: %d\n", snet->sid, ret); 242 else 243 SNET_DBG(pdev, "Reset SNET[%u] device\n", snet->sid); 244 245 return 0; 246 } 247 248 static int snet_reset(struct vdpa_device *vdev) 249 { 250 struct snet *snet = vdpa_to_snet(vdev); 251 252 return snet_reset_dev(snet); 253 } 254 255 static size_t snet_get_config_size(struct vdpa_device *vdev) 256 { 257 struct snet *snet = vdpa_to_snet(vdev); 258 259 return (size_t)snet->cfg->cfg_size; 260 } 261 262 static u64 snet_get_features(struct vdpa_device *vdev) 263 { 264 struct snet *snet = vdpa_to_snet(vdev); 265 266 return snet->cfg->features; 267 } 268 269 static int snet_set_drv_features(struct vdpa_device *vdev, u64 features) 270 { 271 struct snet *snet = vdpa_to_snet(vdev); 272 273 snet->negotiated_features = snet->cfg->features & features; 274 return 0; 275 } 276 277 static u64 snet_get_drv_features(struct vdpa_device *vdev) 278 { 279 struct snet *snet = vdpa_to_snet(vdev); 280 281 return snet->negotiated_features; 282 } 283 284 static u16 snet_get_vq_num_max(struct vdpa_device *vdev) 285 { 286 struct snet *snet = vdpa_to_snet(vdev); 287 288 return (u16)snet->cfg->vq_size; 289 } 290 291 static void snet_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb) 292 { 293 struct snet *snet = vdpa_to_snet(vdev); 294 295 snet->cb.callback = cb->callback; 296 snet->cb.private = cb->private; 297 } 298 299 static u32 snet_get_device_id(struct vdpa_device *vdev) 300 { 301 struct snet *snet = vdpa_to_snet(vdev); 302 303 return snet->cfg->virtio_id; 304 } 305 306 static u32 snet_get_vendor_id(struct vdpa_device *vdev) 307 { 308 return (u32)PCI_VENDOR_ID_SOLIDRUN; 309 } 310 311 static u8 snet_get_status(struct vdpa_device *vdev) 312 { 313 struct snet *snet = vdpa_to_snet(vdev); 314 315 return snet->status; 316 } 317 318 static int snet_write_conf(struct snet *snet) 319 { 320 u32 off, i, tmp; 321 int ret; 322 323 /* No need to write the config twice */ 324 if (snet->dpu_ready) 325 return true; 326 327 /* Snet data : 328 * 329 * General data: SNET_GENERAL_CFG_LEN bytes long 330 * 0 0x4 0x8 0xC 0x10 0x14 0x1C 0x24 331 * | MAGIC NUMBER | CFG VER | SNET SID | NUMBER OF QUEUES | IRQ IDX | FEATURES | RSVD | 332 * 333 * For every VQ: SNET_GENERAL_CFG_VQ_LEN bytes long 334 * 0 0x4 0x8 335 * | VQ SID AND QUEUE SIZE | IRQ Index | 336 * | DESC AREA | 337 * | DEVICE AREA | 338 * | DRIVER AREA | 339 * | VQ STATE (CFG 2+) | RSVD | 340 * 341 * Magic number should be written last, this is the DPU indication that the data is ready 342 */ 343 344 /* Init offset */ 345 off = snet->psnet->cfg.host_cfg_off; 346 347 /* Ignore magic number for now */ 348 off += 4; 349 snet_write32(snet, off, snet->psnet->negotiated_cfg_ver); 350 off += 4; 351 snet_write32(snet, off, snet->sid); 352 off += 4; 353 snet_write32(snet, off, snet->cfg->vq_num); 354 off += 4; 355 snet_write32(snet, off, snet->cfg_irq_idx); 356 off += 4; 357 snet_write64(snet, off, snet->negotiated_features); 358 off += 8; 359 /* Ignore reserved */ 360 off += 8; 361 /* Write VQs */ 362 for (i = 0 ; i < snet->cfg->vq_num ; i++) { 363 tmp = (i << 16) | (snet->vqs[i]->num & 0xFFFF); 364 snet_write32(snet, off, tmp); 365 off += 4; 366 snet_write32(snet, off, snet->vqs[i]->irq_idx); 367 off += 4; 368 snet_write64(snet, off, snet->vqs[i]->desc_area); 369 off += 8; 370 snet_write64(snet, off, snet->vqs[i]->device_area); 371 off += 8; 372 snet_write64(snet, off, snet->vqs[i]->driver_area); 373 off += 8; 374 /* Write VQ state if config version is 2+ */ 375 if (SNET_CFG_VER(snet, 2)) 376 snet_write32(snet, off, *(u32 *)&snet->vqs[i]->vq_state); 377 off += 4; 378 379 /* Ignore reserved */ 380 off += 4; 381 } 382 383 /* Write magic number - data is ready */ 384 snet_write32(snet, snet->psnet->cfg.host_cfg_off, SNET_SIGNATURE); 385 386 /* The DPU will ACK the config by clearing the signature */ 387 ret = readx_poll_timeout(ioread32, snet->bar + snet->psnet->cfg.host_cfg_off, 388 tmp, !tmp, 10, SNET_READ_CFG_TIMEOUT); 389 if (ret) { 390 SNET_ERR(snet->pdev, "Timeout waiting for the DPU to read the config\n"); 391 return false; 392 } 393 394 /* set DPU flag */ 395 snet->dpu_ready = true; 396 397 return true; 398 } 399 400 static int snet_request_irqs(struct pci_dev *pdev, struct snet *snet) 401 { 402 int ret, i, irq; 403 404 /* Request config IRQ */ 405 irq = pci_irq_vector(pdev, snet->cfg_irq_idx); 406 ret = devm_request_irq(&pdev->dev, irq, snet_cfg_irq_hndlr, 0, 407 snet->cfg_irq_name, snet); 408 if (ret) { 409 SNET_ERR(pdev, "Failed to request IRQ\n"); 410 return ret; 411 } 412 snet->cfg_irq = irq; 413 414 /* Request IRQ for every VQ */ 415 for (i = 0; i < snet->cfg->vq_num; i++) { 416 irq = pci_irq_vector(pdev, snet->vqs[i]->irq_idx); 417 ret = devm_request_irq(&pdev->dev, irq, snet_vq_irq_hndlr, 0, 418 snet->vqs[i]->irq_name, snet->vqs[i]); 419 if (ret) { 420 SNET_ERR(pdev, "Failed to request IRQ\n"); 421 return ret; 422 } 423 snet->vqs[i]->irq = irq; 424 } 425 return 0; 426 } 427 428 static void snet_set_status(struct vdpa_device *vdev, u8 status) 429 { 430 struct snet *snet = vdpa_to_snet(vdev); 431 struct psnet *psnet = snet->psnet; 432 struct pci_dev *pdev = snet->pdev; 433 int ret; 434 bool pf_irqs; 435 436 if (status == snet->status) 437 return; 438 439 if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && 440 !(snet->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 441 /* Request IRQs */ 442 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF); 443 ret = snet_request_irqs(pf_irqs ? pdev->physfn : pdev, snet); 444 if (ret) 445 goto set_err; 446 447 /* Write config to the DPU */ 448 if (snet_write_conf(snet)) { 449 SNET_INFO(pdev, "Create SNET[%u] device\n", snet->sid); 450 } else { 451 snet_free_irqs(snet); 452 goto set_err; 453 } 454 } 455 456 /* Save the new status */ 457 snet->status = status; 458 return; 459 460 set_err: 461 snet->status |= VIRTIO_CONFIG_S_FAILED; 462 } 463 464 static void snet_get_config(struct vdpa_device *vdev, unsigned int offset, 465 void *buf, unsigned int len) 466 { 467 struct snet *snet = vdpa_to_snet(vdev); 468 void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset; 469 u8 *buf_ptr = buf; 470 u32 i; 471 472 /* check for offset error */ 473 if (offset + len > snet->cfg->cfg_size) 474 return; 475 476 /* Write into buffer */ 477 for (i = 0; i < len; i++) 478 *buf_ptr++ = ioread8(cfg_ptr + i); 479 } 480 481 static void snet_set_config(struct vdpa_device *vdev, unsigned int offset, 482 const void *buf, unsigned int len) 483 { 484 struct snet *snet = vdpa_to_snet(vdev); 485 void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset; 486 const u8 *buf_ptr = buf; 487 u32 i; 488 489 /* check for offset error */ 490 if (offset + len > snet->cfg->cfg_size) 491 return; 492 493 /* Write into PCI BAR */ 494 for (i = 0; i < len; i++) 495 iowrite8(*buf_ptr++, cfg_ptr + i); 496 } 497 498 static int snet_suspend(struct vdpa_device *vdev) 499 { 500 struct snet *snet = vdpa_to_snet(vdev); 501 int ret; 502 503 ret = snet_suspend_dev(snet); 504 if (ret) 505 SNET_ERR(snet->pdev, "SNET[%u] suspend failed, err: %d\n", snet->sid, ret); 506 else 507 SNET_DBG(snet->pdev, "Suspend SNET[%u] device\n", snet->sid); 508 509 return ret; 510 } 511 512 static int snet_resume(struct vdpa_device *vdev) 513 { 514 struct snet *snet = vdpa_to_snet(vdev); 515 int ret; 516 517 ret = snet_resume_dev(snet); 518 if (ret) 519 SNET_ERR(snet->pdev, "SNET[%u] resume failed, err: %d\n", snet->sid, ret); 520 else 521 SNET_DBG(snet->pdev, "Resume SNET[%u] device\n", snet->sid); 522 523 return ret; 524 } 525 526 static const struct vdpa_config_ops snet_config_ops = { 527 .set_vq_address = snet_set_vq_address, 528 .set_vq_num = snet_set_vq_num, 529 .kick_vq = snet_kick_vq, 530 .kick_vq_with_data = snet_kick_vq_with_data, 531 .set_vq_cb = snet_set_vq_cb, 532 .set_vq_ready = snet_set_vq_ready, 533 .get_vq_ready = snet_get_vq_ready, 534 .set_vq_state = snet_set_vq_state, 535 .get_vq_state = snet_get_vq_state, 536 .get_vq_irq = snet_get_vq_irq, 537 .get_vq_align = snet_get_vq_align, 538 .reset = snet_reset, 539 .get_config_size = snet_get_config_size, 540 .get_device_features = snet_get_features, 541 .set_driver_features = snet_set_drv_features, 542 .get_driver_features = snet_get_drv_features, 543 .get_vq_num_min = snet_get_vq_num_max, 544 .get_vq_num_max = snet_get_vq_num_max, 545 .set_config_cb = snet_set_config_cb, 546 .get_device_id = snet_get_device_id, 547 .get_vendor_id = snet_get_vendor_id, 548 .get_status = snet_get_status, 549 .set_status = snet_set_status, 550 .get_config = snet_get_config, 551 .set_config = snet_set_config, 552 .suspend = snet_suspend, 553 .resume = snet_resume, 554 }; 555 556 static int psnet_open_pf_bar(struct pci_dev *pdev, struct psnet *psnet) 557 { 558 char *name; 559 int ret, i, mask = 0; 560 /* We don't know which BAR will be used to communicate.. 561 * We will map every bar with len > 0. 562 * 563 * Later, we will discover the BAR and unmap all other BARs. 564 */ 565 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 566 if (pci_resource_len(pdev, i)) 567 mask |= (1 << i); 568 } 569 570 /* No BAR can be used.. */ 571 if (!mask) { 572 SNET_ERR(pdev, "Failed to find a PCI BAR\n"); 573 return -ENODEV; 574 } 575 576 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "psnet[%s]-bars", pci_name(pdev)); 577 if (!name) 578 return -ENOMEM; 579 580 ret = pcim_iomap_regions(pdev, mask, name); 581 if (ret) { 582 SNET_ERR(pdev, "Failed to request and map PCI BARs\n"); 583 return ret; 584 } 585 586 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 587 if (mask & (1 << i)) 588 psnet->bars[i] = pcim_iomap_table(pdev)[i]; 589 } 590 591 return 0; 592 } 593 594 static int snet_open_vf_bar(struct pci_dev *pdev, struct snet *snet) 595 { 596 char *name; 597 int ret; 598 599 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "snet[%s]-bars", pci_name(pdev)); 600 if (!name) 601 return -ENOMEM; 602 603 /* Request and map BAR */ 604 ret = pcim_iomap_regions(pdev, BIT(snet->psnet->cfg.vf_bar), name); 605 if (ret) { 606 SNET_ERR(pdev, "Failed to request and map PCI BAR for a VF\n"); 607 return ret; 608 } 609 610 snet->bar = pcim_iomap_table(pdev)[snet->psnet->cfg.vf_bar]; 611 612 return 0; 613 } 614 615 static void snet_free_cfg(struct snet_cfg *cfg) 616 { 617 u32 i; 618 619 if (!cfg->devs) 620 return; 621 622 /* Free devices */ 623 for (i = 0; i < cfg->devices_num; i++) { 624 if (!cfg->devs[i]) 625 break; 626 627 kfree(cfg->devs[i]); 628 } 629 /* Free pointers to devices */ 630 kfree(cfg->devs); 631 } 632 633 /* Detect which BAR is used for communication with the device. */ 634 static int psnet_detect_bar(struct psnet *psnet, u32 off) 635 { 636 unsigned long exit_time; 637 int i; 638 639 exit_time = jiffies + usecs_to_jiffies(SNET_DETECT_TIMEOUT); 640 641 /* SNET DPU will write SNET's signature when the config is ready. */ 642 while (time_before(jiffies, exit_time)) { 643 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 644 /* Is this BAR mapped? */ 645 if (!psnet->bars[i]) 646 continue; 647 648 if (ioread32(psnet->bars[i] + off) == SNET_SIGNATURE) 649 return i; 650 } 651 usleep_range(1000, 10000); 652 } 653 654 return -ENODEV; 655 } 656 657 static void psnet_unmap_unused_bars(struct pci_dev *pdev, struct psnet *psnet) 658 { 659 int i, mask = 0; 660 661 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 662 if (psnet->bars[i] && i != psnet->barno) 663 mask |= (1 << i); 664 } 665 666 if (mask) 667 pcim_iounmap_regions(pdev, mask); 668 } 669 670 /* Read SNET config from PCI BAR */ 671 static int psnet_read_cfg(struct pci_dev *pdev, struct psnet *psnet) 672 { 673 struct snet_cfg *cfg = &psnet->cfg; 674 u32 i, off; 675 int barno; 676 677 /* Move to where the config starts */ 678 off = SNET_CONFIG_OFF; 679 680 /* Find BAR used for communication */ 681 barno = psnet_detect_bar(psnet, off); 682 if (barno < 0) { 683 SNET_ERR(pdev, "SNET config is not ready.\n"); 684 return barno; 685 } 686 687 /* Save used BAR number and unmap all other BARs */ 688 psnet->barno = barno; 689 SNET_DBG(pdev, "Using BAR number %d\n", barno); 690 691 psnet_unmap_unused_bars(pdev, psnet); 692 693 /* load config from BAR */ 694 cfg->key = psnet_read32(psnet, off); 695 off += 4; 696 cfg->cfg_size = psnet_read32(psnet, off); 697 off += 4; 698 cfg->cfg_ver = psnet_read32(psnet, off); 699 off += 4; 700 /* The negotiated config version is the lower one between this driver's config 701 * and the DPU's. 702 */ 703 psnet->negotiated_cfg_ver = min_t(u32, cfg->cfg_ver, SNET_CFG_VERSION); 704 SNET_DBG(pdev, "SNET config version %u\n", psnet->negotiated_cfg_ver); 705 706 cfg->vf_num = psnet_read32(psnet, off); 707 off += 4; 708 cfg->vf_bar = psnet_read32(psnet, off); 709 off += 4; 710 cfg->host_cfg_off = psnet_read32(psnet, off); 711 off += 4; 712 cfg->max_size_host_cfg = psnet_read32(psnet, off); 713 off += 4; 714 cfg->virtio_cfg_off = psnet_read32(psnet, off); 715 off += 4; 716 cfg->kick_off = psnet_read32(psnet, off); 717 off += 4; 718 cfg->hwmon_off = psnet_read32(psnet, off); 719 off += 4; 720 cfg->ctrl_off = psnet_read32(psnet, off); 721 off += 4; 722 cfg->flags = psnet_read32(psnet, off); 723 off += 4; 724 /* Ignore Reserved */ 725 off += sizeof(cfg->rsvd); 726 727 cfg->devices_num = psnet_read32(psnet, off); 728 off += 4; 729 /* Allocate memory to hold pointer to the devices */ 730 cfg->devs = kcalloc(cfg->devices_num, sizeof(void *), GFP_KERNEL); 731 if (!cfg->devs) 732 return -ENOMEM; 733 734 /* Load device configuration from BAR */ 735 for (i = 0; i < cfg->devices_num; i++) { 736 cfg->devs[i] = kzalloc(sizeof(*cfg->devs[i]), GFP_KERNEL); 737 if (!cfg->devs[i]) { 738 snet_free_cfg(cfg); 739 return -ENOMEM; 740 } 741 /* Read device config */ 742 cfg->devs[i]->virtio_id = psnet_read32(psnet, off); 743 off += 4; 744 cfg->devs[i]->vq_num = psnet_read32(psnet, off); 745 off += 4; 746 cfg->devs[i]->vq_size = psnet_read32(psnet, off); 747 off += 4; 748 cfg->devs[i]->vfid = psnet_read32(psnet, off); 749 off += 4; 750 cfg->devs[i]->features = psnet_read64(psnet, off); 751 off += 8; 752 /* Ignore Reserved */ 753 off += sizeof(cfg->devs[i]->rsvd); 754 755 cfg->devs[i]->cfg_size = psnet_read32(psnet, off); 756 off += 4; 757 758 /* Is the config witten to the DPU going to be too big? */ 759 if (SNET_GENERAL_CFG_LEN + SNET_GENERAL_CFG_VQ_LEN * cfg->devs[i]->vq_num > 760 cfg->max_size_host_cfg) { 761 SNET_ERR(pdev, "Failed to read SNET config, the config is too big..\n"); 762 snet_free_cfg(cfg); 763 return -EINVAL; 764 } 765 } 766 return 0; 767 } 768 769 static int psnet_alloc_irq_vector(struct pci_dev *pdev, struct psnet *psnet) 770 { 771 int ret = 0; 772 u32 i, irq_num = 0; 773 774 /* Let's count how many IRQs we need, 1 for every VQ + 1 for config change */ 775 for (i = 0; i < psnet->cfg.devices_num; i++) 776 irq_num += psnet->cfg.devs[i]->vq_num + 1; 777 778 ret = pci_alloc_irq_vectors(pdev, irq_num, irq_num, PCI_IRQ_MSIX); 779 if (ret != irq_num) { 780 SNET_ERR(pdev, "Failed to allocate IRQ vectors\n"); 781 return ret; 782 } 783 SNET_DBG(pdev, "Allocated %u IRQ vectors from physical function\n", irq_num); 784 785 return 0; 786 } 787 788 static int snet_alloc_irq_vector(struct pci_dev *pdev, struct snet_dev_cfg *snet_cfg) 789 { 790 int ret = 0; 791 u32 irq_num; 792 793 /* We want 1 IRQ for every VQ + 1 for config change events */ 794 irq_num = snet_cfg->vq_num + 1; 795 796 ret = pci_alloc_irq_vectors(pdev, irq_num, irq_num, PCI_IRQ_MSIX); 797 if (ret <= 0) { 798 SNET_ERR(pdev, "Failed to allocate IRQ vectors\n"); 799 return ret; 800 } 801 802 return 0; 803 } 804 805 static void snet_free_vqs(struct snet *snet) 806 { 807 u32 i; 808 809 if (!snet->vqs) 810 return; 811 812 for (i = 0 ; i < snet->cfg->vq_num ; i++) { 813 if (!snet->vqs[i]) 814 break; 815 816 kfree(snet->vqs[i]); 817 } 818 kfree(snet->vqs); 819 } 820 821 static int snet_build_vqs(struct snet *snet) 822 { 823 u32 i; 824 /* Allocate the VQ pointers array */ 825 snet->vqs = kcalloc(snet->cfg->vq_num, sizeof(void *), GFP_KERNEL); 826 if (!snet->vqs) 827 return -ENOMEM; 828 829 /* Allocate the VQs */ 830 for (i = 0; i < snet->cfg->vq_num; i++) { 831 snet->vqs[i] = kzalloc(sizeof(*snet->vqs[i]), GFP_KERNEL); 832 if (!snet->vqs[i]) { 833 snet_free_vqs(snet); 834 return -ENOMEM; 835 } 836 /* Reset IRQ num */ 837 snet->vqs[i]->irq = -1; 838 /* VQ serial ID */ 839 snet->vqs[i]->sid = i; 840 /* Kick address - every VQ gets 4B */ 841 snet->vqs[i]->kick_ptr = snet->bar + snet->psnet->cfg.kick_off + 842 snet->vqs[i]->sid * 4; 843 /* Clear kick address for this VQ */ 844 iowrite32(0, snet->vqs[i]->kick_ptr); 845 } 846 return 0; 847 } 848 849 static int psnet_get_next_irq_num(struct psnet *psnet) 850 { 851 int irq; 852 853 spin_lock(&psnet->lock); 854 irq = psnet->next_irq++; 855 spin_unlock(&psnet->lock); 856 857 return irq; 858 } 859 860 static void snet_reserve_irq_idx(struct pci_dev *pdev, struct snet *snet) 861 { 862 struct psnet *psnet = snet->psnet; 863 int i; 864 865 /* one IRQ for every VQ, and one for config changes */ 866 snet->cfg_irq_idx = psnet_get_next_irq_num(psnet); 867 snprintf(snet->cfg_irq_name, SNET_NAME_SIZE, "snet[%s]-cfg[%d]", 868 pci_name(pdev), snet->cfg_irq_idx); 869 870 for (i = 0; i < snet->cfg->vq_num; i++) { 871 /* Get next free IRQ ID */ 872 snet->vqs[i]->irq_idx = psnet_get_next_irq_num(psnet); 873 /* Write IRQ name */ 874 snprintf(snet->vqs[i]->irq_name, SNET_NAME_SIZE, "snet[%s]-vq[%d]", 875 pci_name(pdev), snet->vqs[i]->irq_idx); 876 } 877 } 878 879 /* Find a device config based on virtual function id */ 880 static struct snet_dev_cfg *snet_find_dev_cfg(struct snet_cfg *cfg, u32 vfid) 881 { 882 u32 i; 883 884 for (i = 0; i < cfg->devices_num; i++) { 885 if (cfg->devs[i]->vfid == vfid) 886 return cfg->devs[i]; 887 } 888 /* Oppss.. no config found.. */ 889 return NULL; 890 } 891 892 /* Probe function for a physical PCI function */ 893 static int snet_vdpa_probe_pf(struct pci_dev *pdev) 894 { 895 struct psnet *psnet; 896 int ret = 0; 897 bool pf_irqs = false; 898 899 ret = pcim_enable_device(pdev); 900 if (ret) { 901 SNET_ERR(pdev, "Failed to enable PCI device\n"); 902 return ret; 903 } 904 905 /* Allocate a PCI physical function device */ 906 psnet = kzalloc(sizeof(*psnet), GFP_KERNEL); 907 if (!psnet) 908 return -ENOMEM; 909 910 /* Init PSNET spinlock */ 911 spin_lock_init(&psnet->lock); 912 913 pci_set_master(pdev); 914 pci_set_drvdata(pdev, psnet); 915 916 /* Open SNET MAIN BAR */ 917 ret = psnet_open_pf_bar(pdev, psnet); 918 if (ret) 919 goto free_psnet; 920 921 /* Try to read SNET's config from PCI BAR */ 922 ret = psnet_read_cfg(pdev, psnet); 923 if (ret) 924 goto free_psnet; 925 926 /* If SNET_CFG_FLAG_IRQ_PF flag is set, we should use 927 * PF MSI-X vectors 928 */ 929 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF); 930 931 if (pf_irqs) { 932 ret = psnet_alloc_irq_vector(pdev, psnet); 933 if (ret) 934 goto free_cfg; 935 } 936 937 SNET_DBG(pdev, "Enable %u virtual functions\n", psnet->cfg.vf_num); 938 ret = pci_enable_sriov(pdev, psnet->cfg.vf_num); 939 if (ret) { 940 SNET_ERR(pdev, "Failed to enable SR-IOV\n"); 941 goto free_irq; 942 } 943 944 /* Create HW monitor device */ 945 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_HWMON)) { 946 #if IS_ENABLED(CONFIG_HWMON) 947 psnet_create_hwmon(pdev); 948 #else 949 SNET_WARN(pdev, "Can't start HWMON, CONFIG_HWMON is not enabled\n"); 950 #endif 951 } 952 953 return 0; 954 955 free_irq: 956 if (pf_irqs) 957 pci_free_irq_vectors(pdev); 958 free_cfg: 959 snet_free_cfg(&psnet->cfg); 960 free_psnet: 961 kfree(psnet); 962 return ret; 963 } 964 965 /* Probe function for a virtual PCI function */ 966 static int snet_vdpa_probe_vf(struct pci_dev *pdev) 967 { 968 struct pci_dev *pdev_pf = pdev->physfn; 969 struct psnet *psnet = pci_get_drvdata(pdev_pf); 970 struct snet_dev_cfg *dev_cfg; 971 struct snet *snet; 972 u32 vfid; 973 int ret; 974 bool pf_irqs = false; 975 976 /* Get virtual function id. 977 * (the DPU counts the VFs from 1) 978 */ 979 ret = pci_iov_vf_id(pdev); 980 if (ret < 0) { 981 SNET_ERR(pdev, "Failed to find a VF id\n"); 982 return ret; 983 } 984 vfid = ret + 1; 985 986 /* Find the snet_dev_cfg based on vfid */ 987 dev_cfg = snet_find_dev_cfg(&psnet->cfg, vfid); 988 if (!dev_cfg) { 989 SNET_WARN(pdev, "Failed to find a VF config..\n"); 990 return -ENODEV; 991 } 992 993 /* Which PCI device should allocate the IRQs? 994 * If the SNET_CFG_FLAG_IRQ_PF flag set, the PF device allocates the IRQs 995 */ 996 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF); 997 998 ret = pcim_enable_device(pdev); 999 if (ret) { 1000 SNET_ERR(pdev, "Failed to enable PCI VF device\n"); 1001 return ret; 1002 } 1003 1004 /* Request for MSI-X IRQs */ 1005 if (!pf_irqs) { 1006 ret = snet_alloc_irq_vector(pdev, dev_cfg); 1007 if (ret) 1008 return ret; 1009 } 1010 1011 /* Allocate vdpa device */ 1012 snet = vdpa_alloc_device(struct snet, vdpa, &pdev->dev, &snet_config_ops, 1, 1, NULL, 1013 false); 1014 if (!snet) { 1015 SNET_ERR(pdev, "Failed to allocate a vdpa device\n"); 1016 ret = -ENOMEM; 1017 goto free_irqs; 1018 } 1019 1020 /* Init control mutex and spinlock */ 1021 mutex_init(&snet->ctrl_lock); 1022 spin_lock_init(&snet->ctrl_spinlock); 1023 1024 /* Save pci device pointer */ 1025 snet->pdev = pdev; 1026 snet->psnet = psnet; 1027 snet->cfg = dev_cfg; 1028 snet->dpu_ready = false; 1029 snet->sid = vfid; 1030 /* Reset IRQ value */ 1031 snet->cfg_irq = -1; 1032 1033 ret = snet_open_vf_bar(pdev, snet); 1034 if (ret) 1035 goto put_device; 1036 1037 /* Create a VirtIO config pointer */ 1038 snet->cfg->virtio_cfg = snet->bar + snet->psnet->cfg.virtio_cfg_off; 1039 1040 /* Clear control registers */ 1041 snet_ctrl_clear(snet); 1042 1043 pci_set_master(pdev); 1044 pci_set_drvdata(pdev, snet); 1045 1046 ret = snet_build_vqs(snet); 1047 if (ret) 1048 goto put_device; 1049 1050 /* Reserve IRQ indexes, 1051 * The IRQs may be requested and freed multiple times, 1052 * but the indexes won't change. 1053 */ 1054 snet_reserve_irq_idx(pf_irqs ? pdev_pf : pdev, snet); 1055 1056 /*set DMA device*/ 1057 snet->vdpa.dma_dev = &pdev->dev; 1058 1059 /* Register VDPA device */ 1060 ret = vdpa_register_device(&snet->vdpa, snet->cfg->vq_num); 1061 if (ret) { 1062 SNET_ERR(pdev, "Failed to register vdpa device\n"); 1063 goto free_vqs; 1064 } 1065 1066 return 0; 1067 1068 free_vqs: 1069 snet_free_vqs(snet); 1070 put_device: 1071 put_device(&snet->vdpa.dev); 1072 free_irqs: 1073 if (!pf_irqs) 1074 pci_free_irq_vectors(pdev); 1075 return ret; 1076 } 1077 1078 static int snet_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1079 { 1080 if (pdev->is_virtfn) 1081 return snet_vdpa_probe_vf(pdev); 1082 else 1083 return snet_vdpa_probe_pf(pdev); 1084 } 1085 1086 static void snet_vdpa_remove_pf(struct pci_dev *pdev) 1087 { 1088 struct psnet *psnet = pci_get_drvdata(pdev); 1089 1090 pci_disable_sriov(pdev); 1091 /* If IRQs are allocated from the PF, we should free the IRQs */ 1092 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF)) 1093 pci_free_irq_vectors(pdev); 1094 1095 snet_free_cfg(&psnet->cfg); 1096 kfree(psnet); 1097 } 1098 1099 static void snet_vdpa_remove_vf(struct pci_dev *pdev) 1100 { 1101 struct snet *snet = pci_get_drvdata(pdev); 1102 struct psnet *psnet = snet->psnet; 1103 1104 vdpa_unregister_device(&snet->vdpa); 1105 snet_free_vqs(snet); 1106 /* If IRQs are allocated from the VF, we should free the IRQs */ 1107 if (!PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF)) 1108 pci_free_irq_vectors(pdev); 1109 } 1110 1111 static void snet_vdpa_remove(struct pci_dev *pdev) 1112 { 1113 if (pdev->is_virtfn) 1114 snet_vdpa_remove_vf(pdev); 1115 else 1116 snet_vdpa_remove_pf(pdev); 1117 } 1118 1119 static struct pci_device_id snet_driver_pci_ids[] = { 1120 { PCI_DEVICE_SUB(PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID, 1121 PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID) }, 1122 { 0 }, 1123 }; 1124 1125 MODULE_DEVICE_TABLE(pci, snet_driver_pci_ids); 1126 1127 static struct pci_driver snet_vdpa_driver = { 1128 .name = "snet-vdpa-driver", 1129 .id_table = snet_driver_pci_ids, 1130 .probe = snet_vdpa_probe, 1131 .remove = snet_vdpa_remove, 1132 }; 1133 1134 module_pci_driver(snet_vdpa_driver); 1135 1136 MODULE_AUTHOR("Alvaro Karsz <alvaro.karsz@solid-run.com>"); 1137 MODULE_DESCRIPTION("SolidRun vDPA driver"); 1138 MODULE_LICENSE("GPL v2"); 1139