1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * SolidRun DPU driver for control plane 4 * 5 * Copyright (C) 2022-2023 SolidRun 6 * 7 * Author: Alvaro Karsz <alvaro.karsz@solid-run.com> 8 * 9 */ 10 #include <linux/iopoll.h> 11 12 #include "snet_vdpa.h" 13 14 /* SNET DPU device ID */ 15 #define SNET_DEVICE_ID 0x1000 16 /* SNET signature */ 17 #define SNET_SIGNATURE 0xD0D06363 18 /* Max. config version that we can work with */ 19 #define SNET_CFG_VERSION 0x2 20 /* Queue align */ 21 #define SNET_QUEUE_ALIGNMENT PAGE_SIZE 22 /* Kick value to notify that new data is available */ 23 #define SNET_KICK_VAL 0x1 24 #define SNET_CONFIG_OFF 0x0 25 /* How long we are willing to wait for a SNET device */ 26 #define SNET_DETECT_TIMEOUT 5000000 27 /* How long should we wait for the DPU to read our config */ 28 #define SNET_READ_CFG_TIMEOUT 3000000 29 /* Size of configs written to the DPU */ 30 #define SNET_GENERAL_CFG_LEN 36 31 #define SNET_GENERAL_CFG_VQ_LEN 40 32 33 static struct snet *vdpa_to_snet(struct vdpa_device *vdpa) 34 { 35 return container_of(vdpa, struct snet, vdpa); 36 } 37 38 static irqreturn_t snet_cfg_irq_hndlr(int irq, void *data) 39 { 40 struct snet *snet = data; 41 /* Call callback if any */ 42 if (likely(snet->cb.callback)) 43 return snet->cb.callback(snet->cb.private); 44 45 return IRQ_HANDLED; 46 } 47 48 static irqreturn_t snet_vq_irq_hndlr(int irq, void *data) 49 { 50 struct snet_vq *vq = data; 51 /* Call callback if any */ 52 if (likely(vq->cb.callback)) 53 return vq->cb.callback(vq->cb.private); 54 55 return IRQ_HANDLED; 56 } 57 58 static void snet_free_irqs(struct snet *snet) 59 { 60 struct psnet *psnet = snet->psnet; 61 struct pci_dev *pdev; 62 u32 i; 63 64 /* Which Device allcoated the IRQs? */ 65 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF)) 66 pdev = snet->pdev->physfn; 67 else 68 pdev = snet->pdev; 69 70 /* Free config's IRQ */ 71 if (snet->cfg_irq != -1) { 72 devm_free_irq(&pdev->dev, snet->cfg_irq, snet); 73 snet->cfg_irq = -1; 74 } 75 /* Free VQ IRQs */ 76 for (i = 0; i < snet->cfg->vq_num; i++) { 77 if (snet->vqs[i] && snet->vqs[i]->irq != -1) { 78 devm_free_irq(&pdev->dev, snet->vqs[i]->irq, snet->vqs[i]); 79 snet->vqs[i]->irq = -1; 80 } 81 } 82 83 /* IRQ vectors are freed when the pci remove callback is called */ 84 } 85 86 static int snet_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area, 87 u64 driver_area, u64 device_area) 88 { 89 struct snet *snet = vdpa_to_snet(vdev); 90 /* save received parameters in vqueue sturct */ 91 snet->vqs[idx]->desc_area = desc_area; 92 snet->vqs[idx]->driver_area = driver_area; 93 snet->vqs[idx]->device_area = device_area; 94 95 return 0; 96 } 97 98 static void snet_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) 99 { 100 struct snet *snet = vdpa_to_snet(vdev); 101 /* save num in vqueue */ 102 snet->vqs[idx]->num = num; 103 } 104 105 static void snet_kick_vq(struct vdpa_device *vdev, u16 idx) 106 { 107 struct snet *snet = vdpa_to_snet(vdev); 108 /* not ready - ignore */ 109 if (unlikely(!snet->vqs[idx]->ready)) 110 return; 111 112 iowrite32(SNET_KICK_VAL, snet->vqs[idx]->kick_ptr); 113 } 114 115 static void snet_kick_vq_with_data(struct vdpa_device *vdev, u32 data) 116 { 117 struct snet *snet = vdpa_to_snet(vdev); 118 u16 idx = data & 0xFFFF; 119 120 /* not ready - ignore */ 121 if (unlikely(!snet->vqs[idx]->ready)) 122 return; 123 124 iowrite32((data & 0xFFFF0000) | SNET_KICK_VAL, snet->vqs[idx]->kick_ptr); 125 } 126 127 static void snet_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb) 128 { 129 struct snet *snet = vdpa_to_snet(vdev); 130 131 snet->vqs[idx]->cb.callback = cb->callback; 132 snet->vqs[idx]->cb.private = cb->private; 133 } 134 135 static void snet_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready) 136 { 137 struct snet *snet = vdpa_to_snet(vdev); 138 139 snet->vqs[idx]->ready = ready; 140 } 141 142 static bool snet_get_vq_ready(struct vdpa_device *vdev, u16 idx) 143 { 144 struct snet *snet = vdpa_to_snet(vdev); 145 146 return snet->vqs[idx]->ready; 147 } 148 149 static bool snet_vq_state_is_initial(struct snet *snet, const struct vdpa_vq_state *state) 150 { 151 if (SNET_HAS_FEATURE(snet, VIRTIO_F_RING_PACKED)) { 152 const struct vdpa_vq_state_packed *p = &state->packed; 153 154 if (p->last_avail_counter == 1 && p->last_used_counter == 1 && 155 p->last_avail_idx == 0 && p->last_used_idx == 0) 156 return true; 157 } else { 158 const struct vdpa_vq_state_split *s = &state->split; 159 160 if (s->avail_index == 0) 161 return true; 162 } 163 164 return false; 165 } 166 167 static int snet_set_vq_state(struct vdpa_device *vdev, u16 idx, const struct vdpa_vq_state *state) 168 { 169 struct snet *snet = vdpa_to_snet(vdev); 170 171 /* We can set any state for config version 2+ */ 172 if (SNET_CFG_VER(snet, 2)) { 173 memcpy(&snet->vqs[idx]->vq_state, state, sizeof(*state)); 174 return 0; 175 } 176 177 /* Older config - we can't set the VQ state. 178 * Return 0 only if this is the initial state we use in the DPU. 179 */ 180 if (snet_vq_state_is_initial(snet, state)) 181 return 0; 182 183 return -EOPNOTSUPP; 184 } 185 186 static int snet_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state) 187 { 188 struct snet *snet = vdpa_to_snet(vdev); 189 190 return snet_read_vq_state(snet, idx, state); 191 } 192 193 static int snet_get_vq_irq(struct vdpa_device *vdev, u16 idx) 194 { 195 struct snet *snet = vdpa_to_snet(vdev); 196 197 return snet->vqs[idx]->irq; 198 } 199 200 static u32 snet_get_vq_align(struct vdpa_device *vdev) 201 { 202 return (u32)SNET_QUEUE_ALIGNMENT; 203 } 204 205 static int snet_reset_dev(struct snet *snet) 206 { 207 struct pci_dev *pdev = snet->pdev; 208 int ret = 0; 209 u32 i; 210 211 /* If status is 0, nothing to do */ 212 if (!snet->status) 213 return 0; 214 215 /* If DPU started, destroy it */ 216 if (snet->status & VIRTIO_CONFIG_S_DRIVER_OK) 217 ret = snet_destroy_dev(snet); 218 219 /* Clear VQs */ 220 for (i = 0; i < snet->cfg->vq_num; i++) { 221 if (!snet->vqs[i]) 222 continue; 223 snet->vqs[i]->cb.callback = NULL; 224 snet->vqs[i]->cb.private = NULL; 225 snet->vqs[i]->desc_area = 0; 226 snet->vqs[i]->device_area = 0; 227 snet->vqs[i]->driver_area = 0; 228 snet->vqs[i]->ready = false; 229 } 230 231 /* Clear config callback */ 232 snet->cb.callback = NULL; 233 snet->cb.private = NULL; 234 /* Free IRQs */ 235 snet_free_irqs(snet); 236 /* Reset status */ 237 snet->status = 0; 238 snet->dpu_ready = false; 239 240 if (ret) 241 SNET_WARN(pdev, "Incomplete reset to SNET[%u] device, err: %d\n", snet->sid, ret); 242 else 243 SNET_DBG(pdev, "Reset SNET[%u] device\n", snet->sid); 244 245 return 0; 246 } 247 248 static int snet_reset(struct vdpa_device *vdev) 249 { 250 struct snet *snet = vdpa_to_snet(vdev); 251 252 return snet_reset_dev(snet); 253 } 254 255 static size_t snet_get_config_size(struct vdpa_device *vdev) 256 { 257 struct snet *snet = vdpa_to_snet(vdev); 258 259 return (size_t)snet->cfg->cfg_size; 260 } 261 262 static u64 snet_get_features(struct vdpa_device *vdev) 263 { 264 struct snet *snet = vdpa_to_snet(vdev); 265 266 return snet->cfg->features; 267 } 268 269 static int snet_set_drv_features(struct vdpa_device *vdev, u64 features) 270 { 271 struct snet *snet = vdpa_to_snet(vdev); 272 273 snet->negotiated_features = snet->cfg->features & features; 274 return 0; 275 } 276 277 static u64 snet_get_drv_features(struct vdpa_device *vdev) 278 { 279 struct snet *snet = vdpa_to_snet(vdev); 280 281 return snet->negotiated_features; 282 } 283 284 static u16 snet_get_vq_num_max(struct vdpa_device *vdev) 285 { 286 struct snet *snet = vdpa_to_snet(vdev); 287 288 return (u16)snet->cfg->vq_size; 289 } 290 291 static void snet_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb) 292 { 293 struct snet *snet = vdpa_to_snet(vdev); 294 295 snet->cb.callback = cb->callback; 296 snet->cb.private = cb->private; 297 } 298 299 static u32 snet_get_device_id(struct vdpa_device *vdev) 300 { 301 struct snet *snet = vdpa_to_snet(vdev); 302 303 return snet->cfg->virtio_id; 304 } 305 306 static u32 snet_get_vendor_id(struct vdpa_device *vdev) 307 { 308 return (u32)PCI_VENDOR_ID_SOLIDRUN; 309 } 310 311 static u8 snet_get_status(struct vdpa_device *vdev) 312 { 313 struct snet *snet = vdpa_to_snet(vdev); 314 315 return snet->status; 316 } 317 318 static int snet_write_conf(struct snet *snet) 319 { 320 u32 off, i, tmp; 321 int ret; 322 323 /* No need to write the config twice */ 324 if (snet->dpu_ready) 325 return true; 326 327 /* Snet data : 328 * 329 * General data: SNET_GENERAL_CFG_LEN bytes long 330 * 0 0x4 0x8 0xC 0x10 0x14 0x1C 0x24 331 * | MAGIC NUMBER | CFG VER | SNET SID | NUMBER OF QUEUES | IRQ IDX | FEATURES | RSVD | 332 * 333 * For every VQ: SNET_GENERAL_CFG_VQ_LEN bytes long 334 * 0 0x4 0x8 335 * | VQ SID AND QUEUE SIZE | IRQ Index | 336 * | DESC AREA | 337 * | DEVICE AREA | 338 * | DRIVER AREA | 339 * | VQ STATE (CFG 2+) | RSVD | 340 * 341 * Magic number should be written last, this is the DPU indication that the data is ready 342 */ 343 344 /* Init offset */ 345 off = snet->psnet->cfg.host_cfg_off; 346 347 /* Ignore magic number for now */ 348 off += 4; 349 snet_write32(snet, off, snet->psnet->negotiated_cfg_ver); 350 off += 4; 351 snet_write32(snet, off, snet->sid); 352 off += 4; 353 snet_write32(snet, off, snet->cfg->vq_num); 354 off += 4; 355 snet_write32(snet, off, snet->cfg_irq_idx); 356 off += 4; 357 snet_write64(snet, off, snet->negotiated_features); 358 off += 8; 359 /* Ignore reserved */ 360 off += 8; 361 /* Write VQs */ 362 for (i = 0 ; i < snet->cfg->vq_num ; i++) { 363 tmp = (i << 16) | (snet->vqs[i]->num & 0xFFFF); 364 snet_write32(snet, off, tmp); 365 off += 4; 366 snet_write32(snet, off, snet->vqs[i]->irq_idx); 367 off += 4; 368 snet_write64(snet, off, snet->vqs[i]->desc_area); 369 off += 8; 370 snet_write64(snet, off, snet->vqs[i]->device_area); 371 off += 8; 372 snet_write64(snet, off, snet->vqs[i]->driver_area); 373 off += 8; 374 /* Write VQ state if config version is 2+ */ 375 if (SNET_CFG_VER(snet, 2)) 376 snet_write32(snet, off, *(u32 *)&snet->vqs[i]->vq_state); 377 off += 4; 378 379 /* Ignore reserved */ 380 off += 4; 381 } 382 383 /* Write magic number - data is ready */ 384 snet_write32(snet, snet->psnet->cfg.host_cfg_off, SNET_SIGNATURE); 385 386 /* The DPU will ACK the config by clearing the signature */ 387 ret = readx_poll_timeout(ioread32, snet->bar + snet->psnet->cfg.host_cfg_off, 388 tmp, !tmp, 10, SNET_READ_CFG_TIMEOUT); 389 if (ret) { 390 SNET_ERR(snet->pdev, "Timeout waiting for the DPU to read the config\n"); 391 return false; 392 } 393 394 /* set DPU flag */ 395 snet->dpu_ready = true; 396 397 return true; 398 } 399 400 static int snet_request_irqs(struct pci_dev *pdev, struct snet *snet) 401 { 402 int ret, i, irq; 403 404 /* Request config IRQ */ 405 irq = pci_irq_vector(pdev, snet->cfg_irq_idx); 406 ret = devm_request_irq(&pdev->dev, irq, snet_cfg_irq_hndlr, 0, 407 snet->cfg_irq_name, snet); 408 if (ret) { 409 SNET_ERR(pdev, "Failed to request IRQ\n"); 410 return ret; 411 } 412 snet->cfg_irq = irq; 413 414 /* Request IRQ for every VQ */ 415 for (i = 0; i < snet->cfg->vq_num; i++) { 416 irq = pci_irq_vector(pdev, snet->vqs[i]->irq_idx); 417 ret = devm_request_irq(&pdev->dev, irq, snet_vq_irq_hndlr, 0, 418 snet->vqs[i]->irq_name, snet->vqs[i]); 419 if (ret) { 420 SNET_ERR(pdev, "Failed to request IRQ\n"); 421 return ret; 422 } 423 snet->vqs[i]->irq = irq; 424 } 425 return 0; 426 } 427 428 static void snet_set_status(struct vdpa_device *vdev, u8 status) 429 { 430 struct snet *snet = vdpa_to_snet(vdev); 431 struct psnet *psnet = snet->psnet; 432 struct pci_dev *pdev = snet->pdev; 433 int ret; 434 bool pf_irqs; 435 436 if (status == snet->status) 437 return; 438 439 if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && 440 !(snet->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 441 /* Request IRQs */ 442 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF); 443 ret = snet_request_irqs(pf_irqs ? pdev->physfn : pdev, snet); 444 if (ret) 445 goto set_err; 446 447 /* Write config to the DPU */ 448 if (snet_write_conf(snet)) { 449 SNET_INFO(pdev, "Create SNET[%u] device\n", snet->sid); 450 } else { 451 snet_free_irqs(snet); 452 goto set_err; 453 } 454 } 455 456 /* Save the new status */ 457 snet->status = status; 458 return; 459 460 set_err: 461 snet->status |= VIRTIO_CONFIG_S_FAILED; 462 } 463 464 static void snet_get_config(struct vdpa_device *vdev, unsigned int offset, 465 void *buf, unsigned int len) 466 { 467 struct snet *snet = vdpa_to_snet(vdev); 468 void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset; 469 u8 *buf_ptr = buf; 470 u32 i; 471 472 /* check for offset error */ 473 if (offset + len > snet->cfg->cfg_size) 474 return; 475 476 /* Write into buffer */ 477 for (i = 0; i < len; i++) 478 *buf_ptr++ = ioread8(cfg_ptr + i); 479 } 480 481 static void snet_set_config(struct vdpa_device *vdev, unsigned int offset, 482 const void *buf, unsigned int len) 483 { 484 struct snet *snet = vdpa_to_snet(vdev); 485 void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset; 486 const u8 *buf_ptr = buf; 487 u32 i; 488 489 /* check for offset error */ 490 if (offset + len > snet->cfg->cfg_size) 491 return; 492 493 /* Write into PCI BAR */ 494 for (i = 0; i < len; i++) 495 iowrite8(*buf_ptr++, cfg_ptr + i); 496 } 497 498 static int snet_suspend(struct vdpa_device *vdev) 499 { 500 struct snet *snet = vdpa_to_snet(vdev); 501 int ret; 502 503 ret = snet_suspend_dev(snet); 504 if (ret) 505 SNET_ERR(snet->pdev, "SNET[%u] suspend failed, err: %d\n", snet->sid, ret); 506 else 507 SNET_DBG(snet->pdev, "Suspend SNET[%u] device\n", snet->sid); 508 509 return ret; 510 } 511 512 static int snet_resume(struct vdpa_device *vdev) 513 { 514 struct snet *snet = vdpa_to_snet(vdev); 515 int ret; 516 517 ret = snet_resume_dev(snet); 518 if (ret) 519 SNET_ERR(snet->pdev, "SNET[%u] resume failed, err: %d\n", snet->sid, ret); 520 else 521 SNET_DBG(snet->pdev, "Resume SNET[%u] device\n", snet->sid); 522 523 return ret; 524 } 525 526 static const struct vdpa_config_ops snet_config_ops = { 527 .set_vq_address = snet_set_vq_address, 528 .set_vq_num = snet_set_vq_num, 529 .kick_vq = snet_kick_vq, 530 .kick_vq_with_data = snet_kick_vq_with_data, 531 .set_vq_cb = snet_set_vq_cb, 532 .set_vq_ready = snet_set_vq_ready, 533 .get_vq_ready = snet_get_vq_ready, 534 .set_vq_state = snet_set_vq_state, 535 .get_vq_state = snet_get_vq_state, 536 .get_vq_irq = snet_get_vq_irq, 537 .get_vq_align = snet_get_vq_align, 538 .reset = snet_reset, 539 .get_config_size = snet_get_config_size, 540 .get_device_features = snet_get_features, 541 .set_driver_features = snet_set_drv_features, 542 .get_driver_features = snet_get_drv_features, 543 .get_vq_num_min = snet_get_vq_num_max, 544 .get_vq_num_max = snet_get_vq_num_max, 545 .set_config_cb = snet_set_config_cb, 546 .get_device_id = snet_get_device_id, 547 .get_vendor_id = snet_get_vendor_id, 548 .get_status = snet_get_status, 549 .set_status = snet_set_status, 550 .get_config = snet_get_config, 551 .set_config = snet_set_config, 552 .suspend = snet_suspend, 553 .resume = snet_resume, 554 }; 555 556 static int psnet_open_pf_bar(struct pci_dev *pdev, struct psnet *psnet) 557 { 558 char name[50]; 559 int ret, i, mask = 0; 560 /* We don't know which BAR will be used to communicate.. 561 * We will map every bar with len > 0. 562 * 563 * Later, we will discover the BAR and unmap all other BARs. 564 */ 565 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 566 if (pci_resource_len(pdev, i)) 567 mask |= (1 << i); 568 } 569 570 /* No BAR can be used.. */ 571 if (!mask) { 572 SNET_ERR(pdev, "Failed to find a PCI BAR\n"); 573 return -ENODEV; 574 } 575 576 snprintf(name, sizeof(name), "psnet[%s]-bars", pci_name(pdev)); 577 ret = pcim_iomap_regions(pdev, mask, name); 578 if (ret) { 579 SNET_ERR(pdev, "Failed to request and map PCI BARs\n"); 580 return ret; 581 } 582 583 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 584 if (mask & (1 << i)) 585 psnet->bars[i] = pcim_iomap_table(pdev)[i]; 586 } 587 588 return 0; 589 } 590 591 static int snet_open_vf_bar(struct pci_dev *pdev, struct snet *snet) 592 { 593 char name[50]; 594 int ret; 595 596 snprintf(name, sizeof(name), "snet[%s]-bar", pci_name(pdev)); 597 /* Request and map BAR */ 598 ret = pcim_iomap_regions(pdev, BIT(snet->psnet->cfg.vf_bar), name); 599 if (ret) { 600 SNET_ERR(pdev, "Failed to request and map PCI BAR for a VF\n"); 601 return ret; 602 } 603 604 snet->bar = pcim_iomap_table(pdev)[snet->psnet->cfg.vf_bar]; 605 606 return 0; 607 } 608 609 static void snet_free_cfg(struct snet_cfg *cfg) 610 { 611 u32 i; 612 613 if (!cfg->devs) 614 return; 615 616 /* Free devices */ 617 for (i = 0; i < cfg->devices_num; i++) { 618 if (!cfg->devs[i]) 619 break; 620 621 kfree(cfg->devs[i]); 622 } 623 /* Free pointers to devices */ 624 kfree(cfg->devs); 625 } 626 627 /* Detect which BAR is used for communication with the device. */ 628 static int psnet_detect_bar(struct psnet *psnet, u32 off) 629 { 630 unsigned long exit_time; 631 int i; 632 633 exit_time = jiffies + usecs_to_jiffies(SNET_DETECT_TIMEOUT); 634 635 /* SNET DPU will write SNET's signature when the config is ready. */ 636 while (time_before(jiffies, exit_time)) { 637 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 638 /* Is this BAR mapped? */ 639 if (!psnet->bars[i]) 640 continue; 641 642 if (ioread32(psnet->bars[i] + off) == SNET_SIGNATURE) 643 return i; 644 } 645 usleep_range(1000, 10000); 646 } 647 648 return -ENODEV; 649 } 650 651 static void psnet_unmap_unused_bars(struct pci_dev *pdev, struct psnet *psnet) 652 { 653 int i, mask = 0; 654 655 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 656 if (psnet->bars[i] && i != psnet->barno) 657 mask |= (1 << i); 658 } 659 660 if (mask) 661 pcim_iounmap_regions(pdev, mask); 662 } 663 664 /* Read SNET config from PCI BAR */ 665 static int psnet_read_cfg(struct pci_dev *pdev, struct psnet *psnet) 666 { 667 struct snet_cfg *cfg = &psnet->cfg; 668 u32 i, off; 669 int barno; 670 671 /* Move to where the config starts */ 672 off = SNET_CONFIG_OFF; 673 674 /* Find BAR used for communication */ 675 barno = psnet_detect_bar(psnet, off); 676 if (barno < 0) { 677 SNET_ERR(pdev, "SNET config is not ready.\n"); 678 return barno; 679 } 680 681 /* Save used BAR number and unmap all other BARs */ 682 psnet->barno = barno; 683 SNET_DBG(pdev, "Using BAR number %d\n", barno); 684 685 psnet_unmap_unused_bars(pdev, psnet); 686 687 /* load config from BAR */ 688 cfg->key = psnet_read32(psnet, off); 689 off += 4; 690 cfg->cfg_size = psnet_read32(psnet, off); 691 off += 4; 692 cfg->cfg_ver = psnet_read32(psnet, off); 693 off += 4; 694 /* The negotiated config version is the lower one between this driver's config 695 * and the DPU's. 696 */ 697 psnet->negotiated_cfg_ver = min_t(u32, cfg->cfg_ver, SNET_CFG_VERSION); 698 SNET_DBG(pdev, "SNET config version %u\n", psnet->negotiated_cfg_ver); 699 700 cfg->vf_num = psnet_read32(psnet, off); 701 off += 4; 702 cfg->vf_bar = psnet_read32(psnet, off); 703 off += 4; 704 cfg->host_cfg_off = psnet_read32(psnet, off); 705 off += 4; 706 cfg->max_size_host_cfg = psnet_read32(psnet, off); 707 off += 4; 708 cfg->virtio_cfg_off = psnet_read32(psnet, off); 709 off += 4; 710 cfg->kick_off = psnet_read32(psnet, off); 711 off += 4; 712 cfg->hwmon_off = psnet_read32(psnet, off); 713 off += 4; 714 cfg->ctrl_off = psnet_read32(psnet, off); 715 off += 4; 716 cfg->flags = psnet_read32(psnet, off); 717 off += 4; 718 /* Ignore Reserved */ 719 off += sizeof(cfg->rsvd); 720 721 cfg->devices_num = psnet_read32(psnet, off); 722 off += 4; 723 /* Allocate memory to hold pointer to the devices */ 724 cfg->devs = kcalloc(cfg->devices_num, sizeof(void *), GFP_KERNEL); 725 if (!cfg->devs) 726 return -ENOMEM; 727 728 /* Load device configuration from BAR */ 729 for (i = 0; i < cfg->devices_num; i++) { 730 cfg->devs[i] = kzalloc(sizeof(*cfg->devs[i]), GFP_KERNEL); 731 if (!cfg->devs[i]) { 732 snet_free_cfg(cfg); 733 return -ENOMEM; 734 } 735 /* Read device config */ 736 cfg->devs[i]->virtio_id = psnet_read32(psnet, off); 737 off += 4; 738 cfg->devs[i]->vq_num = psnet_read32(psnet, off); 739 off += 4; 740 cfg->devs[i]->vq_size = psnet_read32(psnet, off); 741 off += 4; 742 cfg->devs[i]->vfid = psnet_read32(psnet, off); 743 off += 4; 744 cfg->devs[i]->features = psnet_read64(psnet, off); 745 off += 8; 746 /* Ignore Reserved */ 747 off += sizeof(cfg->devs[i]->rsvd); 748 749 cfg->devs[i]->cfg_size = psnet_read32(psnet, off); 750 off += 4; 751 752 /* Is the config witten to the DPU going to be too big? */ 753 if (SNET_GENERAL_CFG_LEN + SNET_GENERAL_CFG_VQ_LEN * cfg->devs[i]->vq_num > 754 cfg->max_size_host_cfg) { 755 SNET_ERR(pdev, "Failed to read SNET config, the config is too big..\n"); 756 snet_free_cfg(cfg); 757 return -EINVAL; 758 } 759 } 760 return 0; 761 } 762 763 static int psnet_alloc_irq_vector(struct pci_dev *pdev, struct psnet *psnet) 764 { 765 int ret = 0; 766 u32 i, irq_num = 0; 767 768 /* Let's count how many IRQs we need, 1 for every VQ + 1 for config change */ 769 for (i = 0; i < psnet->cfg.devices_num; i++) 770 irq_num += psnet->cfg.devs[i]->vq_num + 1; 771 772 ret = pci_alloc_irq_vectors(pdev, irq_num, irq_num, PCI_IRQ_MSIX); 773 if (ret != irq_num) { 774 SNET_ERR(pdev, "Failed to allocate IRQ vectors\n"); 775 return ret; 776 } 777 SNET_DBG(pdev, "Allocated %u IRQ vectors from physical function\n", irq_num); 778 779 return 0; 780 } 781 782 static int snet_alloc_irq_vector(struct pci_dev *pdev, struct snet_dev_cfg *snet_cfg) 783 { 784 int ret = 0; 785 u32 irq_num; 786 787 /* We want 1 IRQ for every VQ + 1 for config change events */ 788 irq_num = snet_cfg->vq_num + 1; 789 790 ret = pci_alloc_irq_vectors(pdev, irq_num, irq_num, PCI_IRQ_MSIX); 791 if (ret <= 0) { 792 SNET_ERR(pdev, "Failed to allocate IRQ vectors\n"); 793 return ret; 794 } 795 796 return 0; 797 } 798 799 static void snet_free_vqs(struct snet *snet) 800 { 801 u32 i; 802 803 if (!snet->vqs) 804 return; 805 806 for (i = 0 ; i < snet->cfg->vq_num ; i++) { 807 if (!snet->vqs[i]) 808 break; 809 810 kfree(snet->vqs[i]); 811 } 812 kfree(snet->vqs); 813 } 814 815 static int snet_build_vqs(struct snet *snet) 816 { 817 u32 i; 818 /* Allocate the VQ pointers array */ 819 snet->vqs = kcalloc(snet->cfg->vq_num, sizeof(void *), GFP_KERNEL); 820 if (!snet->vqs) 821 return -ENOMEM; 822 823 /* Allocate the VQs */ 824 for (i = 0; i < snet->cfg->vq_num; i++) { 825 snet->vqs[i] = kzalloc(sizeof(*snet->vqs[i]), GFP_KERNEL); 826 if (!snet->vqs[i]) { 827 snet_free_vqs(snet); 828 return -ENOMEM; 829 } 830 /* Reset IRQ num */ 831 snet->vqs[i]->irq = -1; 832 /* VQ serial ID */ 833 snet->vqs[i]->sid = i; 834 /* Kick address - every VQ gets 4B */ 835 snet->vqs[i]->kick_ptr = snet->bar + snet->psnet->cfg.kick_off + 836 snet->vqs[i]->sid * 4; 837 /* Clear kick address for this VQ */ 838 iowrite32(0, snet->vqs[i]->kick_ptr); 839 } 840 return 0; 841 } 842 843 static int psnet_get_next_irq_num(struct psnet *psnet) 844 { 845 int irq; 846 847 spin_lock(&psnet->lock); 848 irq = psnet->next_irq++; 849 spin_unlock(&psnet->lock); 850 851 return irq; 852 } 853 854 static void snet_reserve_irq_idx(struct pci_dev *pdev, struct snet *snet) 855 { 856 struct psnet *psnet = snet->psnet; 857 int i; 858 859 /* one IRQ for every VQ, and one for config changes */ 860 snet->cfg_irq_idx = psnet_get_next_irq_num(psnet); 861 snprintf(snet->cfg_irq_name, SNET_NAME_SIZE, "snet[%s]-cfg[%d]", 862 pci_name(pdev), snet->cfg_irq_idx); 863 864 for (i = 0; i < snet->cfg->vq_num; i++) { 865 /* Get next free IRQ ID */ 866 snet->vqs[i]->irq_idx = psnet_get_next_irq_num(psnet); 867 /* Write IRQ name */ 868 snprintf(snet->vqs[i]->irq_name, SNET_NAME_SIZE, "snet[%s]-vq[%d]", 869 pci_name(pdev), snet->vqs[i]->irq_idx); 870 } 871 } 872 873 /* Find a device config based on virtual function id */ 874 static struct snet_dev_cfg *snet_find_dev_cfg(struct snet_cfg *cfg, u32 vfid) 875 { 876 u32 i; 877 878 for (i = 0; i < cfg->devices_num; i++) { 879 if (cfg->devs[i]->vfid == vfid) 880 return cfg->devs[i]; 881 } 882 /* Oppss.. no config found.. */ 883 return NULL; 884 } 885 886 /* Probe function for a physical PCI function */ 887 static int snet_vdpa_probe_pf(struct pci_dev *pdev) 888 { 889 struct psnet *psnet; 890 int ret = 0; 891 bool pf_irqs = false; 892 893 ret = pcim_enable_device(pdev); 894 if (ret) { 895 SNET_ERR(pdev, "Failed to enable PCI device\n"); 896 return ret; 897 } 898 899 /* Allocate a PCI physical function device */ 900 psnet = kzalloc(sizeof(*psnet), GFP_KERNEL); 901 if (!psnet) 902 return -ENOMEM; 903 904 /* Init PSNET spinlock */ 905 spin_lock_init(&psnet->lock); 906 907 pci_set_master(pdev); 908 pci_set_drvdata(pdev, psnet); 909 910 /* Open SNET MAIN BAR */ 911 ret = psnet_open_pf_bar(pdev, psnet); 912 if (ret) 913 goto free_psnet; 914 915 /* Try to read SNET's config from PCI BAR */ 916 ret = psnet_read_cfg(pdev, psnet); 917 if (ret) 918 goto free_psnet; 919 920 /* If SNET_CFG_FLAG_IRQ_PF flag is set, we should use 921 * PF MSI-X vectors 922 */ 923 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF); 924 925 if (pf_irqs) { 926 ret = psnet_alloc_irq_vector(pdev, psnet); 927 if (ret) 928 goto free_cfg; 929 } 930 931 SNET_DBG(pdev, "Enable %u virtual functions\n", psnet->cfg.vf_num); 932 ret = pci_enable_sriov(pdev, psnet->cfg.vf_num); 933 if (ret) { 934 SNET_ERR(pdev, "Failed to enable SR-IOV\n"); 935 goto free_irq; 936 } 937 938 /* Create HW monitor device */ 939 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_HWMON)) { 940 #if IS_ENABLED(CONFIG_HWMON) 941 psnet_create_hwmon(pdev); 942 #else 943 SNET_WARN(pdev, "Can't start HWMON, CONFIG_HWMON is not enabled\n"); 944 #endif 945 } 946 947 return 0; 948 949 free_irq: 950 if (pf_irqs) 951 pci_free_irq_vectors(pdev); 952 free_cfg: 953 snet_free_cfg(&psnet->cfg); 954 free_psnet: 955 kfree(psnet); 956 return ret; 957 } 958 959 /* Probe function for a virtual PCI function */ 960 static int snet_vdpa_probe_vf(struct pci_dev *pdev) 961 { 962 struct pci_dev *pdev_pf = pdev->physfn; 963 struct psnet *psnet = pci_get_drvdata(pdev_pf); 964 struct snet_dev_cfg *dev_cfg; 965 struct snet *snet; 966 u32 vfid; 967 int ret; 968 bool pf_irqs = false; 969 970 /* Get virtual function id. 971 * (the DPU counts the VFs from 1) 972 */ 973 ret = pci_iov_vf_id(pdev); 974 if (ret < 0) { 975 SNET_ERR(pdev, "Failed to find a VF id\n"); 976 return ret; 977 } 978 vfid = ret + 1; 979 980 /* Find the snet_dev_cfg based on vfid */ 981 dev_cfg = snet_find_dev_cfg(&psnet->cfg, vfid); 982 if (!dev_cfg) { 983 SNET_WARN(pdev, "Failed to find a VF config..\n"); 984 return -ENODEV; 985 } 986 987 /* Which PCI device should allocate the IRQs? 988 * If the SNET_CFG_FLAG_IRQ_PF flag set, the PF device allocates the IRQs 989 */ 990 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF); 991 992 ret = pcim_enable_device(pdev); 993 if (ret) { 994 SNET_ERR(pdev, "Failed to enable PCI VF device\n"); 995 return ret; 996 } 997 998 /* Request for MSI-X IRQs */ 999 if (!pf_irqs) { 1000 ret = snet_alloc_irq_vector(pdev, dev_cfg); 1001 if (ret) 1002 return ret; 1003 } 1004 1005 /* Allocate vdpa device */ 1006 snet = vdpa_alloc_device(struct snet, vdpa, &pdev->dev, &snet_config_ops, 1, 1, NULL, 1007 false); 1008 if (!snet) { 1009 SNET_ERR(pdev, "Failed to allocate a vdpa device\n"); 1010 ret = -ENOMEM; 1011 goto free_irqs; 1012 } 1013 1014 /* Init control mutex and spinlock */ 1015 mutex_init(&snet->ctrl_lock); 1016 spin_lock_init(&snet->ctrl_spinlock); 1017 1018 /* Save pci device pointer */ 1019 snet->pdev = pdev; 1020 snet->psnet = psnet; 1021 snet->cfg = dev_cfg; 1022 snet->dpu_ready = false; 1023 snet->sid = vfid; 1024 /* Reset IRQ value */ 1025 snet->cfg_irq = -1; 1026 1027 ret = snet_open_vf_bar(pdev, snet); 1028 if (ret) 1029 goto put_device; 1030 1031 /* Create a VirtIO config pointer */ 1032 snet->cfg->virtio_cfg = snet->bar + snet->psnet->cfg.virtio_cfg_off; 1033 1034 /* Clear control registers */ 1035 snet_ctrl_clear(snet); 1036 1037 pci_set_master(pdev); 1038 pci_set_drvdata(pdev, snet); 1039 1040 ret = snet_build_vqs(snet); 1041 if (ret) 1042 goto put_device; 1043 1044 /* Reserve IRQ indexes, 1045 * The IRQs may be requested and freed multiple times, 1046 * but the indexes won't change. 1047 */ 1048 snet_reserve_irq_idx(pf_irqs ? pdev_pf : pdev, snet); 1049 1050 /*set DMA device*/ 1051 snet->vdpa.dma_dev = &pdev->dev; 1052 1053 /* Register VDPA device */ 1054 ret = vdpa_register_device(&snet->vdpa, snet->cfg->vq_num); 1055 if (ret) { 1056 SNET_ERR(pdev, "Failed to register vdpa device\n"); 1057 goto free_vqs; 1058 } 1059 1060 return 0; 1061 1062 free_vqs: 1063 snet_free_vqs(snet); 1064 put_device: 1065 put_device(&snet->vdpa.dev); 1066 free_irqs: 1067 if (!pf_irqs) 1068 pci_free_irq_vectors(pdev); 1069 return ret; 1070 } 1071 1072 static int snet_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1073 { 1074 if (pdev->is_virtfn) 1075 return snet_vdpa_probe_vf(pdev); 1076 else 1077 return snet_vdpa_probe_pf(pdev); 1078 } 1079 1080 static void snet_vdpa_remove_pf(struct pci_dev *pdev) 1081 { 1082 struct psnet *psnet = pci_get_drvdata(pdev); 1083 1084 pci_disable_sriov(pdev); 1085 /* If IRQs are allocated from the PF, we should free the IRQs */ 1086 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF)) 1087 pci_free_irq_vectors(pdev); 1088 1089 snet_free_cfg(&psnet->cfg); 1090 kfree(psnet); 1091 } 1092 1093 static void snet_vdpa_remove_vf(struct pci_dev *pdev) 1094 { 1095 struct snet *snet = pci_get_drvdata(pdev); 1096 struct psnet *psnet = snet->psnet; 1097 1098 vdpa_unregister_device(&snet->vdpa); 1099 snet_free_vqs(snet); 1100 /* If IRQs are allocated from the VF, we should free the IRQs */ 1101 if (!PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF)) 1102 pci_free_irq_vectors(pdev); 1103 } 1104 1105 static void snet_vdpa_remove(struct pci_dev *pdev) 1106 { 1107 if (pdev->is_virtfn) 1108 snet_vdpa_remove_vf(pdev); 1109 else 1110 snet_vdpa_remove_pf(pdev); 1111 } 1112 1113 static struct pci_device_id snet_driver_pci_ids[] = { 1114 { PCI_DEVICE_SUB(PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID, 1115 PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID) }, 1116 { 0 }, 1117 }; 1118 1119 MODULE_DEVICE_TABLE(pci, snet_driver_pci_ids); 1120 1121 static struct pci_driver snet_vdpa_driver = { 1122 .name = "snet-vdpa-driver", 1123 .id_table = snet_driver_pci_ids, 1124 .probe = snet_vdpa_probe, 1125 .remove = snet_vdpa_remove, 1126 }; 1127 1128 module_pci_driver(snet_vdpa_driver); 1129 1130 MODULE_AUTHOR("Alvaro Karsz <alvaro.karsz@solid-run.com>"); 1131 MODULE_DESCRIPTION("SolidRun vDPA driver"); 1132 MODULE_LICENSE("GPL v2"); 1133