1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * SolidRun DPU driver for control plane 4 * 5 * Copyright (C) 2022 SolidRun 6 * 7 * Author: Alvaro Karsz <alvaro.karsz@solid-run.com> 8 * 9 */ 10 #include <linux/iopoll.h> 11 12 #include "snet_vdpa.h" 13 14 /* SNET DPU device ID */ 15 #define SNET_DEVICE_ID 0x1000 16 /* SNET signature */ 17 #define SNET_SIGNATURE 0xD0D06363 18 /* Max. config version that we can work with */ 19 #define SNET_CFG_VERSION 0x1 20 /* Queue align */ 21 #define SNET_QUEUE_ALIGNMENT PAGE_SIZE 22 /* Kick value to notify that new data is available */ 23 #define SNET_KICK_VAL 0x1 24 #define SNET_CONFIG_OFF 0x0 25 /* ACK timeout for a message */ 26 #define SNET_ACK_TIMEOUT 2000000 27 /* How long we are willing to wait for a SNET device */ 28 #define SNET_DETECT_TIMEOUT 5000000 29 /* How long should we wait for the DPU to read our config */ 30 #define SNET_READ_CFG_TIMEOUT 3000000 31 /* Size of configs written to the DPU */ 32 #define SNET_GENERAL_CFG_LEN 36 33 #define SNET_GENERAL_CFG_VQ_LEN 40 34 35 enum snet_msg { 36 SNET_MSG_DESTROY = 1, 37 }; 38 39 static struct snet *vdpa_to_snet(struct vdpa_device *vdpa) 40 { 41 return container_of(vdpa, struct snet, vdpa); 42 } 43 44 static int snet_wait_for_msg_ack(struct snet *snet) 45 { 46 struct pci_dev *pdev = snet->pdev; 47 int ret; 48 u32 val; 49 50 /* The DPU will clear the messages offset once messages 51 * are processed. 52 */ 53 ret = readx_poll_timeout(ioread32, snet->bar + snet->psnet->cfg.msg_off, 54 val, !val, 10, SNET_ACK_TIMEOUT); 55 if (ret) 56 SNET_WARN(pdev, "Timeout waiting for message ACK\n"); 57 58 return ret; 59 } 60 61 /* Sends a message to the DPU. 62 * If blocking is set, the function will return once the 63 * message was processed by the DPU (or timeout). 64 */ 65 static int snet_send_msg(struct snet *snet, u32 msg, bool blocking) 66 { 67 int ret = 0; 68 69 /* Make sure the DPU acked last message before issuing a new one */ 70 ret = snet_wait_for_msg_ack(snet); 71 if (ret) 72 return ret; 73 74 /* Write the message */ 75 snet_write32(snet, snet->psnet->cfg.msg_off, msg); 76 77 if (blocking) 78 ret = snet_wait_for_msg_ack(snet); 79 else /* If non-blocking, flush the write by issuing a read */ 80 snet_read32(snet, snet->psnet->cfg.msg_off); 81 82 return ret; 83 } 84 85 static irqreturn_t snet_cfg_irq_hndlr(int irq, void *data) 86 { 87 struct snet *snet = data; 88 /* Call callback if any */ 89 if (snet->cb.callback) 90 return snet->cb.callback(snet->cb.private); 91 92 return IRQ_HANDLED; 93 } 94 95 static irqreturn_t snet_vq_irq_hndlr(int irq, void *data) 96 { 97 struct snet_vq *vq = data; 98 /* Call callback if any */ 99 if (vq->cb.callback) 100 return vq->cb.callback(vq->cb.private); 101 102 return IRQ_HANDLED; 103 } 104 105 static void snet_free_irqs(struct snet *snet) 106 { 107 struct psnet *psnet = snet->psnet; 108 struct pci_dev *pdev; 109 u32 i; 110 111 /* Which Device allcoated the IRQs? */ 112 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF)) 113 pdev = snet->pdev->physfn; 114 else 115 pdev = snet->pdev; 116 117 /* Free config's IRQ */ 118 if (snet->cfg_irq != -1) { 119 devm_free_irq(&pdev->dev, snet->cfg_irq, snet); 120 snet->cfg_irq = -1; 121 } 122 /* Free VQ IRQs */ 123 for (i = 0; i < snet->cfg->vq_num; i++) { 124 if (snet->vqs[i] && snet->vqs[i]->irq != -1) { 125 devm_free_irq(&pdev->dev, snet->vqs[i]->irq, snet->vqs[i]); 126 snet->vqs[i]->irq = -1; 127 } 128 } 129 130 /* IRQ vectors are freed when the pci remove callback is called */ 131 } 132 133 static int snet_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area, 134 u64 driver_area, u64 device_area) 135 { 136 struct snet *snet = vdpa_to_snet(vdev); 137 /* save received parameters in vqueue sturct */ 138 snet->vqs[idx]->desc_area = desc_area; 139 snet->vqs[idx]->driver_area = driver_area; 140 snet->vqs[idx]->device_area = device_area; 141 142 return 0; 143 } 144 145 static void snet_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) 146 { 147 struct snet *snet = vdpa_to_snet(vdev); 148 /* save num in vqueue */ 149 snet->vqs[idx]->num = num; 150 } 151 152 static void snet_kick_vq(struct vdpa_device *vdev, u16 idx) 153 { 154 struct snet *snet = vdpa_to_snet(vdev); 155 /* not ready - ignore */ 156 if (!snet->vqs[idx]->ready) 157 return; 158 159 iowrite32(SNET_KICK_VAL, snet->vqs[idx]->kick_ptr); 160 } 161 162 static void snet_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb) 163 { 164 struct snet *snet = vdpa_to_snet(vdev); 165 166 snet->vqs[idx]->cb.callback = cb->callback; 167 snet->vqs[idx]->cb.private = cb->private; 168 } 169 170 static void snet_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready) 171 { 172 struct snet *snet = vdpa_to_snet(vdev); 173 174 snet->vqs[idx]->ready = ready; 175 } 176 177 static bool snet_get_vq_ready(struct vdpa_device *vdev, u16 idx) 178 { 179 struct snet *snet = vdpa_to_snet(vdev); 180 181 return snet->vqs[idx]->ready; 182 } 183 184 static int snet_set_vq_state(struct vdpa_device *vdev, u16 idx, const struct vdpa_vq_state *state) 185 { 186 struct snet *snet = vdpa_to_snet(vdev); 187 /* Setting the VQ state is not supported. 188 * If the asked state is the same as the initial one 189 * we can ignore it. 190 */ 191 if (SNET_HAS_FEATURE(snet, VIRTIO_F_RING_PACKED)) { 192 const struct vdpa_vq_state_packed *p = &state->packed; 193 194 if (p->last_avail_counter == 1 && p->last_used_counter == 1 && 195 p->last_avail_idx == 0 && p->last_used_idx == 0) 196 return 0; 197 } else { 198 const struct vdpa_vq_state_split *s = &state->split; 199 200 if (s->avail_index == 0) 201 return 0; 202 } 203 204 return -EOPNOTSUPP; 205 } 206 207 static int snet_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state) 208 { 209 /* Not supported */ 210 return -EOPNOTSUPP; 211 } 212 213 static int snet_get_vq_irq(struct vdpa_device *vdev, u16 idx) 214 { 215 struct snet *snet = vdpa_to_snet(vdev); 216 217 return snet->vqs[idx]->irq; 218 } 219 220 static u32 snet_get_vq_align(struct vdpa_device *vdev) 221 { 222 return (u32)SNET_QUEUE_ALIGNMENT; 223 } 224 225 static int snet_reset_dev(struct snet *snet) 226 { 227 struct pci_dev *pdev = snet->pdev; 228 int ret = 0; 229 u32 i; 230 231 /* If status is 0, nothing to do */ 232 if (!snet->status) 233 return 0; 234 235 /* If DPU started, send a destroy message */ 236 if (snet->status & VIRTIO_CONFIG_S_DRIVER_OK) 237 ret = snet_send_msg(snet, SNET_MSG_DESTROY, true); 238 239 /* Clear VQs */ 240 for (i = 0; i < snet->cfg->vq_num; i++) { 241 if (!snet->vqs[i]) 242 continue; 243 snet->vqs[i]->cb.callback = NULL; 244 snet->vqs[i]->cb.private = NULL; 245 snet->vqs[i]->desc_area = 0; 246 snet->vqs[i]->device_area = 0; 247 snet->vqs[i]->driver_area = 0; 248 snet->vqs[i]->ready = false; 249 } 250 251 /* Clear config callback */ 252 snet->cb.callback = NULL; 253 snet->cb.private = NULL; 254 /* Free IRQs */ 255 snet_free_irqs(snet); 256 /* Reset status */ 257 snet->status = 0; 258 snet->dpu_ready = false; 259 260 if (ret) 261 SNET_WARN(pdev, "Incomplete reset to SNET[%u] device\n", snet->sid); 262 else 263 SNET_DBG(pdev, "Reset SNET[%u] device\n", snet->sid); 264 265 return 0; 266 } 267 268 static int snet_reset(struct vdpa_device *vdev) 269 { 270 struct snet *snet = vdpa_to_snet(vdev); 271 272 return snet_reset_dev(snet); 273 } 274 275 static size_t snet_get_config_size(struct vdpa_device *vdev) 276 { 277 struct snet *snet = vdpa_to_snet(vdev); 278 279 return (size_t)snet->cfg->cfg_size; 280 } 281 282 static u64 snet_get_features(struct vdpa_device *vdev) 283 { 284 struct snet *snet = vdpa_to_snet(vdev); 285 286 return snet->cfg->features; 287 } 288 289 static int snet_set_drv_features(struct vdpa_device *vdev, u64 features) 290 { 291 struct snet *snet = vdpa_to_snet(vdev); 292 293 snet->negotiated_features = snet->cfg->features & features; 294 return 0; 295 } 296 297 static u64 snet_get_drv_features(struct vdpa_device *vdev) 298 { 299 struct snet *snet = vdpa_to_snet(vdev); 300 301 return snet->negotiated_features; 302 } 303 304 static u16 snet_get_vq_num_max(struct vdpa_device *vdev) 305 { 306 struct snet *snet = vdpa_to_snet(vdev); 307 308 return (u16)snet->cfg->vq_size; 309 } 310 311 static void snet_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb) 312 { 313 struct snet *snet = vdpa_to_snet(vdev); 314 315 snet->cb.callback = cb->callback; 316 snet->cb.private = cb->private; 317 } 318 319 static u32 snet_get_device_id(struct vdpa_device *vdev) 320 { 321 struct snet *snet = vdpa_to_snet(vdev); 322 323 return snet->cfg->virtio_id; 324 } 325 326 static u32 snet_get_vendor_id(struct vdpa_device *vdev) 327 { 328 return (u32)PCI_VENDOR_ID_SOLIDRUN; 329 } 330 331 static u8 snet_get_status(struct vdpa_device *vdev) 332 { 333 struct snet *snet = vdpa_to_snet(vdev); 334 335 return snet->status; 336 } 337 338 static int snet_write_conf(struct snet *snet) 339 { 340 u32 off, i, tmp; 341 int ret; 342 343 /* No need to write the config twice */ 344 if (snet->dpu_ready) 345 return true; 346 347 /* Snet data : 348 * 349 * General data: SNET_GENERAL_CFG_LEN bytes long 350 * 0 0x4 0x8 0xC 0x10 0x14 0x1C 0x24 351 * | MAGIC NUMBER | CFG VER | SNET SID | NUMBER OF QUEUES | IRQ IDX | FEATURES | RSVD | 352 * 353 * For every VQ: SNET_GENERAL_CFG_VQ_LEN bytes long 354 * 0 0x4 0x8 355 * | VQ SID AND QUEUE SIZE | IRQ Index | 356 * | DESC AREA | 357 * | DEVICE AREA | 358 * | DRIVER AREA | 359 * | RESERVED | 360 * 361 * Magic number should be written last, this is the DPU indication that the data is ready 362 */ 363 364 /* Init offset */ 365 off = snet->psnet->cfg.host_cfg_off; 366 367 /* Ignore magic number for now */ 368 off += 4; 369 snet_write32(snet, off, snet->psnet->negotiated_cfg_ver); 370 off += 4; 371 snet_write32(snet, off, snet->sid); 372 off += 4; 373 snet_write32(snet, off, snet->cfg->vq_num); 374 off += 4; 375 snet_write32(snet, off, snet->cfg_irq_idx); 376 off += 4; 377 snet_write64(snet, off, snet->negotiated_features); 378 off += 8; 379 /* Ignore reserved */ 380 off += 8; 381 /* Write VQs */ 382 for (i = 0 ; i < snet->cfg->vq_num ; i++) { 383 tmp = (i << 16) | (snet->vqs[i]->num & 0xFFFF); 384 snet_write32(snet, off, tmp); 385 off += 4; 386 snet_write32(snet, off, snet->vqs[i]->irq_idx); 387 off += 4; 388 snet_write64(snet, off, snet->vqs[i]->desc_area); 389 off += 8; 390 snet_write64(snet, off, snet->vqs[i]->device_area); 391 off += 8; 392 snet_write64(snet, off, snet->vqs[i]->driver_area); 393 off += 8; 394 /* Ignore reserved */ 395 off += 8; 396 } 397 398 /* Clear snet messages address for this device */ 399 snet_write32(snet, snet->psnet->cfg.msg_off, 0); 400 /* Write magic number - data is ready */ 401 snet_write32(snet, snet->psnet->cfg.host_cfg_off, SNET_SIGNATURE); 402 403 /* The DPU will ACK the config by clearing the signature */ 404 ret = readx_poll_timeout(ioread32, snet->bar + snet->psnet->cfg.host_cfg_off, 405 tmp, !tmp, 10, SNET_READ_CFG_TIMEOUT); 406 if (ret) { 407 SNET_ERR(snet->pdev, "Timeout waiting for the DPU to read the config\n"); 408 return false; 409 } 410 411 /* set DPU flag */ 412 snet->dpu_ready = true; 413 414 return true; 415 } 416 417 static int snet_request_irqs(struct pci_dev *pdev, struct snet *snet) 418 { 419 int ret, i, irq; 420 421 /* Request config IRQ */ 422 irq = pci_irq_vector(pdev, snet->cfg_irq_idx); 423 ret = devm_request_irq(&pdev->dev, irq, snet_cfg_irq_hndlr, 0, 424 snet->cfg_irq_name, snet); 425 if (ret) { 426 SNET_ERR(pdev, "Failed to request IRQ\n"); 427 return ret; 428 } 429 snet->cfg_irq = irq; 430 431 /* Request IRQ for every VQ */ 432 for (i = 0; i < snet->cfg->vq_num; i++) { 433 irq = pci_irq_vector(pdev, snet->vqs[i]->irq_idx); 434 ret = devm_request_irq(&pdev->dev, irq, snet_vq_irq_hndlr, 0, 435 snet->vqs[i]->irq_name, snet->vqs[i]); 436 if (ret) { 437 SNET_ERR(pdev, "Failed to request IRQ\n"); 438 return ret; 439 } 440 snet->vqs[i]->irq = irq; 441 } 442 return 0; 443 } 444 445 static void snet_set_status(struct vdpa_device *vdev, u8 status) 446 { 447 struct snet *snet = vdpa_to_snet(vdev); 448 struct psnet *psnet = snet->psnet; 449 struct pci_dev *pdev = snet->pdev; 450 int ret; 451 bool pf_irqs; 452 453 if (status == snet->status) 454 return; 455 456 if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && 457 !(snet->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 458 /* Request IRQs */ 459 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF); 460 ret = snet_request_irqs(pf_irqs ? pdev->physfn : pdev, snet); 461 if (ret) 462 goto set_err; 463 464 /* Write config to the DPU */ 465 if (snet_write_conf(snet)) { 466 SNET_INFO(pdev, "Create SNET[%u] device\n", snet->sid); 467 } else { 468 snet_free_irqs(snet); 469 goto set_err; 470 } 471 } 472 473 /* Save the new status */ 474 snet->status = status; 475 return; 476 477 set_err: 478 snet->status |= VIRTIO_CONFIG_S_FAILED; 479 } 480 481 static void snet_get_config(struct vdpa_device *vdev, unsigned int offset, 482 void *buf, unsigned int len) 483 { 484 struct snet *snet = vdpa_to_snet(vdev); 485 void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset; 486 u8 *buf_ptr = buf; 487 u32 i; 488 489 /* check for offset error */ 490 if (offset + len > snet->cfg->cfg_size) 491 return; 492 493 /* Write into buffer */ 494 for (i = 0; i < len; i++) 495 *buf_ptr++ = ioread8(cfg_ptr + i); 496 } 497 498 static void snet_set_config(struct vdpa_device *vdev, unsigned int offset, 499 const void *buf, unsigned int len) 500 { 501 struct snet *snet = vdpa_to_snet(vdev); 502 void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset; 503 const u8 *buf_ptr = buf; 504 u32 i; 505 506 /* check for offset error */ 507 if (offset + len > snet->cfg->cfg_size) 508 return; 509 510 /* Write into PCI BAR */ 511 for (i = 0; i < len; i++) 512 iowrite8(*buf_ptr++, cfg_ptr + i); 513 } 514 515 static const struct vdpa_config_ops snet_config_ops = { 516 .set_vq_address = snet_set_vq_address, 517 .set_vq_num = snet_set_vq_num, 518 .kick_vq = snet_kick_vq, 519 .set_vq_cb = snet_set_vq_cb, 520 .set_vq_ready = snet_set_vq_ready, 521 .get_vq_ready = snet_get_vq_ready, 522 .set_vq_state = snet_set_vq_state, 523 .get_vq_state = snet_get_vq_state, 524 .get_vq_irq = snet_get_vq_irq, 525 .get_vq_align = snet_get_vq_align, 526 .reset = snet_reset, 527 .get_config_size = snet_get_config_size, 528 .get_device_features = snet_get_features, 529 .set_driver_features = snet_set_drv_features, 530 .get_driver_features = snet_get_drv_features, 531 .get_vq_num_min = snet_get_vq_num_max, 532 .get_vq_num_max = snet_get_vq_num_max, 533 .set_config_cb = snet_set_config_cb, 534 .get_device_id = snet_get_device_id, 535 .get_vendor_id = snet_get_vendor_id, 536 .get_status = snet_get_status, 537 .set_status = snet_set_status, 538 .get_config = snet_get_config, 539 .set_config = snet_set_config, 540 }; 541 542 static int psnet_open_pf_bar(struct pci_dev *pdev, struct psnet *psnet) 543 { 544 char name[50]; 545 int ret, i, mask = 0; 546 /* We don't know which BAR will be used to communicate.. 547 * We will map every bar with len > 0. 548 * 549 * Later, we will discover the BAR and unmap all other BARs. 550 */ 551 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 552 if (pci_resource_len(pdev, i)) 553 mask |= (1 << i); 554 } 555 556 /* No BAR can be used.. */ 557 if (!mask) { 558 SNET_ERR(pdev, "Failed to find a PCI BAR\n"); 559 return -ENODEV; 560 } 561 562 snprintf(name, sizeof(name), "psnet[%s]-bars", pci_name(pdev)); 563 ret = pcim_iomap_regions(pdev, mask, name); 564 if (ret) { 565 SNET_ERR(pdev, "Failed to request and map PCI BARs\n"); 566 return ret; 567 } 568 569 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 570 if (mask & (1 << i)) 571 psnet->bars[i] = pcim_iomap_table(pdev)[i]; 572 } 573 574 return 0; 575 } 576 577 static int snet_open_vf_bar(struct pci_dev *pdev, struct snet *snet) 578 { 579 char name[50]; 580 int ret; 581 582 snprintf(name, sizeof(name), "snet[%s]-bar", pci_name(pdev)); 583 /* Request and map BAR */ 584 ret = pcim_iomap_regions(pdev, BIT(snet->psnet->cfg.vf_bar), name); 585 if (ret) { 586 SNET_ERR(pdev, "Failed to request and map PCI BAR for a VF\n"); 587 return ret; 588 } 589 590 snet->bar = pcim_iomap_table(pdev)[snet->psnet->cfg.vf_bar]; 591 592 return 0; 593 } 594 595 static void snet_free_cfg(struct snet_cfg *cfg) 596 { 597 u32 i; 598 599 if (!cfg->devs) 600 return; 601 602 /* Free devices */ 603 for (i = 0; i < cfg->devices_num; i++) { 604 if (!cfg->devs[i]) 605 break; 606 607 kfree(cfg->devs[i]); 608 } 609 /* Free pointers to devices */ 610 kfree(cfg->devs); 611 } 612 613 /* Detect which BAR is used for communication with the device. */ 614 static int psnet_detect_bar(struct psnet *psnet, u32 off) 615 { 616 unsigned long exit_time; 617 int i; 618 619 exit_time = jiffies + usecs_to_jiffies(SNET_DETECT_TIMEOUT); 620 621 /* SNET DPU will write SNET's signature when the config is ready. */ 622 while (time_before(jiffies, exit_time)) { 623 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 624 /* Is this BAR mapped? */ 625 if (!psnet->bars[i]) 626 continue; 627 628 if (ioread32(psnet->bars[i] + off) == SNET_SIGNATURE) 629 return i; 630 } 631 usleep_range(1000, 10000); 632 } 633 634 return -ENODEV; 635 } 636 637 static void psnet_unmap_unused_bars(struct pci_dev *pdev, struct psnet *psnet) 638 { 639 int i, mask = 0; 640 641 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 642 if (psnet->bars[i] && i != psnet->barno) 643 mask |= (1 << i); 644 } 645 646 if (mask) 647 pcim_iounmap_regions(pdev, mask); 648 } 649 650 /* Read SNET config from PCI BAR */ 651 static int psnet_read_cfg(struct pci_dev *pdev, struct psnet *psnet) 652 { 653 struct snet_cfg *cfg = &psnet->cfg; 654 u32 i, off; 655 int barno; 656 657 /* Move to where the config starts */ 658 off = SNET_CONFIG_OFF; 659 660 /* Find BAR used for communication */ 661 barno = psnet_detect_bar(psnet, off); 662 if (barno < 0) { 663 SNET_ERR(pdev, "SNET config is not ready.\n"); 664 return barno; 665 } 666 667 /* Save used BAR number and unmap all other BARs */ 668 psnet->barno = barno; 669 SNET_DBG(pdev, "Using BAR number %d\n", barno); 670 671 psnet_unmap_unused_bars(pdev, psnet); 672 673 /* load config from BAR */ 674 cfg->key = psnet_read32(psnet, off); 675 off += 4; 676 cfg->cfg_size = psnet_read32(psnet, off); 677 off += 4; 678 cfg->cfg_ver = psnet_read32(psnet, off); 679 off += 4; 680 /* The negotiated config version is the lower one between this driver's config 681 * and the DPU's. 682 */ 683 psnet->negotiated_cfg_ver = min_t(u32, cfg->cfg_ver, SNET_CFG_VERSION); 684 SNET_DBG(pdev, "SNET config version %u\n", psnet->negotiated_cfg_ver); 685 686 cfg->vf_num = psnet_read32(psnet, off); 687 off += 4; 688 cfg->vf_bar = psnet_read32(psnet, off); 689 off += 4; 690 cfg->host_cfg_off = psnet_read32(psnet, off); 691 off += 4; 692 cfg->max_size_host_cfg = psnet_read32(psnet, off); 693 off += 4; 694 cfg->virtio_cfg_off = psnet_read32(psnet, off); 695 off += 4; 696 cfg->kick_off = psnet_read32(psnet, off); 697 off += 4; 698 cfg->hwmon_off = psnet_read32(psnet, off); 699 off += 4; 700 cfg->msg_off = psnet_read32(psnet, off); 701 off += 4; 702 cfg->flags = psnet_read32(psnet, off); 703 off += 4; 704 /* Ignore Reserved */ 705 off += sizeof(cfg->rsvd); 706 707 cfg->devices_num = psnet_read32(psnet, off); 708 off += 4; 709 /* Allocate memory to hold pointer to the devices */ 710 cfg->devs = kcalloc(cfg->devices_num, sizeof(void *), GFP_KERNEL); 711 if (!cfg->devs) 712 return -ENOMEM; 713 714 /* Load device configuration from BAR */ 715 for (i = 0; i < cfg->devices_num; i++) { 716 cfg->devs[i] = kzalloc(sizeof(*cfg->devs[i]), GFP_KERNEL); 717 if (!cfg->devs[i]) { 718 snet_free_cfg(cfg); 719 return -ENOMEM; 720 } 721 /* Read device config */ 722 cfg->devs[i]->virtio_id = psnet_read32(psnet, off); 723 off += 4; 724 cfg->devs[i]->vq_num = psnet_read32(psnet, off); 725 off += 4; 726 cfg->devs[i]->vq_size = psnet_read32(psnet, off); 727 off += 4; 728 cfg->devs[i]->vfid = psnet_read32(psnet, off); 729 off += 4; 730 cfg->devs[i]->features = psnet_read64(psnet, off); 731 off += 8; 732 /* Ignore Reserved */ 733 off += sizeof(cfg->devs[i]->rsvd); 734 735 cfg->devs[i]->cfg_size = psnet_read32(psnet, off); 736 off += 4; 737 738 /* Is the config witten to the DPU going to be too big? */ 739 if (SNET_GENERAL_CFG_LEN + SNET_GENERAL_CFG_VQ_LEN * cfg->devs[i]->vq_num > 740 cfg->max_size_host_cfg) { 741 SNET_ERR(pdev, "Failed to read SNET config, the config is too big..\n"); 742 snet_free_cfg(cfg); 743 return -EINVAL; 744 } 745 } 746 return 0; 747 } 748 749 static int psnet_alloc_irq_vector(struct pci_dev *pdev, struct psnet *psnet) 750 { 751 int ret = 0; 752 u32 i, irq_num = 0; 753 754 /* Let's count how many IRQs we need, 1 for every VQ + 1 for config change */ 755 for (i = 0; i < psnet->cfg.devices_num; i++) 756 irq_num += psnet->cfg.devs[i]->vq_num + 1; 757 758 ret = pci_alloc_irq_vectors(pdev, irq_num, irq_num, PCI_IRQ_MSIX); 759 if (ret != irq_num) { 760 SNET_ERR(pdev, "Failed to allocate IRQ vectors\n"); 761 return ret; 762 } 763 SNET_DBG(pdev, "Allocated %u IRQ vectors from physical function\n", irq_num); 764 765 return 0; 766 } 767 768 static int snet_alloc_irq_vector(struct pci_dev *pdev, struct snet_dev_cfg *snet_cfg) 769 { 770 int ret = 0; 771 u32 irq_num; 772 773 /* We want 1 IRQ for every VQ + 1 for config change events */ 774 irq_num = snet_cfg->vq_num + 1; 775 776 ret = pci_alloc_irq_vectors(pdev, irq_num, irq_num, PCI_IRQ_MSIX); 777 if (ret <= 0) { 778 SNET_ERR(pdev, "Failed to allocate IRQ vectors\n"); 779 return ret; 780 } 781 782 return 0; 783 } 784 785 static void snet_free_vqs(struct snet *snet) 786 { 787 u32 i; 788 789 if (!snet->vqs) 790 return; 791 792 for (i = 0 ; i < snet->cfg->vq_num ; i++) { 793 if (!snet->vqs[i]) 794 break; 795 796 kfree(snet->vqs[i]); 797 } 798 kfree(snet->vqs); 799 } 800 801 static int snet_build_vqs(struct snet *snet) 802 { 803 u32 i; 804 /* Allocate the VQ pointers array */ 805 snet->vqs = kcalloc(snet->cfg->vq_num, sizeof(void *), GFP_KERNEL); 806 if (!snet->vqs) 807 return -ENOMEM; 808 809 /* Allocate the VQs */ 810 for (i = 0; i < snet->cfg->vq_num; i++) { 811 snet->vqs[i] = kzalloc(sizeof(*snet->vqs[i]), GFP_KERNEL); 812 if (!snet->vqs[i]) { 813 snet_free_vqs(snet); 814 return -ENOMEM; 815 } 816 /* Reset IRQ num */ 817 snet->vqs[i]->irq = -1; 818 /* VQ serial ID */ 819 snet->vqs[i]->sid = i; 820 /* Kick address - every VQ gets 4B */ 821 snet->vqs[i]->kick_ptr = snet->bar + snet->psnet->cfg.kick_off + 822 snet->vqs[i]->sid * 4; 823 /* Clear kick address for this VQ */ 824 iowrite32(0, snet->vqs[i]->kick_ptr); 825 } 826 return 0; 827 } 828 829 static int psnet_get_next_irq_num(struct psnet *psnet) 830 { 831 int irq; 832 833 spin_lock(&psnet->lock); 834 irq = psnet->next_irq++; 835 spin_unlock(&psnet->lock); 836 837 return irq; 838 } 839 840 static void snet_reserve_irq_idx(struct pci_dev *pdev, struct snet *snet) 841 { 842 struct psnet *psnet = snet->psnet; 843 int i; 844 845 /* one IRQ for every VQ, and one for config changes */ 846 snet->cfg_irq_idx = psnet_get_next_irq_num(psnet); 847 snprintf(snet->cfg_irq_name, SNET_NAME_SIZE, "snet[%s]-cfg[%d]", 848 pci_name(pdev), snet->cfg_irq_idx); 849 850 for (i = 0; i < snet->cfg->vq_num; i++) { 851 /* Get next free IRQ ID */ 852 snet->vqs[i]->irq_idx = psnet_get_next_irq_num(psnet); 853 /* Write IRQ name */ 854 snprintf(snet->vqs[i]->irq_name, SNET_NAME_SIZE, "snet[%s]-vq[%d]", 855 pci_name(pdev), snet->vqs[i]->irq_idx); 856 } 857 } 858 859 /* Find a device config based on virtual function id */ 860 static struct snet_dev_cfg *snet_find_dev_cfg(struct snet_cfg *cfg, u32 vfid) 861 { 862 u32 i; 863 864 for (i = 0; i < cfg->devices_num; i++) { 865 if (cfg->devs[i]->vfid == vfid) 866 return cfg->devs[i]; 867 } 868 /* Oppss.. no config found.. */ 869 return NULL; 870 } 871 872 /* Probe function for a physical PCI function */ 873 static int snet_vdpa_probe_pf(struct pci_dev *pdev) 874 { 875 struct psnet *psnet; 876 int ret = 0; 877 bool pf_irqs = false; 878 879 ret = pcim_enable_device(pdev); 880 if (ret) { 881 SNET_ERR(pdev, "Failed to enable PCI device\n"); 882 return ret; 883 } 884 885 /* Allocate a PCI physical function device */ 886 psnet = kzalloc(sizeof(*psnet), GFP_KERNEL); 887 if (!psnet) 888 return -ENOMEM; 889 890 /* Init PSNET spinlock */ 891 spin_lock_init(&psnet->lock); 892 893 pci_set_master(pdev); 894 pci_set_drvdata(pdev, psnet); 895 896 /* Open SNET MAIN BAR */ 897 ret = psnet_open_pf_bar(pdev, psnet); 898 if (ret) 899 goto free_psnet; 900 901 /* Try to read SNET's config from PCI BAR */ 902 ret = psnet_read_cfg(pdev, psnet); 903 if (ret) 904 goto free_psnet; 905 906 /* If SNET_CFG_FLAG_IRQ_PF flag is set, we should use 907 * PF MSI-X vectors 908 */ 909 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF); 910 911 if (pf_irqs) { 912 ret = psnet_alloc_irq_vector(pdev, psnet); 913 if (ret) 914 goto free_cfg; 915 } 916 917 SNET_DBG(pdev, "Enable %u virtual functions\n", psnet->cfg.vf_num); 918 ret = pci_enable_sriov(pdev, psnet->cfg.vf_num); 919 if (ret) { 920 SNET_ERR(pdev, "Failed to enable SR-IOV\n"); 921 goto free_irq; 922 } 923 924 /* Create HW monitor device */ 925 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_HWMON)) { 926 #if IS_ENABLED(CONFIG_HWMON) 927 psnet_create_hwmon(pdev); 928 #else 929 SNET_WARN(pdev, "Can't start HWMON, CONFIG_HWMON is not enabled\n"); 930 #endif 931 } 932 933 return 0; 934 935 free_irq: 936 if (pf_irqs) 937 pci_free_irq_vectors(pdev); 938 free_cfg: 939 snet_free_cfg(&psnet->cfg); 940 free_psnet: 941 kfree(psnet); 942 return ret; 943 } 944 945 /* Probe function for a virtual PCI function */ 946 static int snet_vdpa_probe_vf(struct pci_dev *pdev) 947 { 948 struct pci_dev *pdev_pf = pdev->physfn; 949 struct psnet *psnet = pci_get_drvdata(pdev_pf); 950 struct snet_dev_cfg *dev_cfg; 951 struct snet *snet; 952 u32 vfid; 953 int ret; 954 bool pf_irqs = false; 955 956 /* Get virtual function id. 957 * (the DPU counts the VFs from 1) 958 */ 959 ret = pci_iov_vf_id(pdev); 960 if (ret < 0) { 961 SNET_ERR(pdev, "Failed to find a VF id\n"); 962 return ret; 963 } 964 vfid = ret + 1; 965 966 /* Find the snet_dev_cfg based on vfid */ 967 dev_cfg = snet_find_dev_cfg(&psnet->cfg, vfid); 968 if (!dev_cfg) { 969 SNET_WARN(pdev, "Failed to find a VF config..\n"); 970 return -ENODEV; 971 } 972 973 /* Which PCI device should allocate the IRQs? 974 * If the SNET_CFG_FLAG_IRQ_PF flag set, the PF device allocates the IRQs 975 */ 976 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF); 977 978 ret = pcim_enable_device(pdev); 979 if (ret) { 980 SNET_ERR(pdev, "Failed to enable PCI VF device\n"); 981 return ret; 982 } 983 984 /* Request for MSI-X IRQs */ 985 if (!pf_irqs) { 986 ret = snet_alloc_irq_vector(pdev, dev_cfg); 987 if (ret) 988 return ret; 989 } 990 991 /* Allocate vdpa device */ 992 snet = vdpa_alloc_device(struct snet, vdpa, &pdev->dev, &snet_config_ops, 1, 1, NULL, 993 false); 994 if (!snet) { 995 SNET_ERR(pdev, "Failed to allocate a vdpa device\n"); 996 ret = -ENOMEM; 997 goto free_irqs; 998 } 999 1000 /* Save pci device pointer */ 1001 snet->pdev = pdev; 1002 snet->psnet = psnet; 1003 snet->cfg = dev_cfg; 1004 snet->dpu_ready = false; 1005 snet->sid = vfid; 1006 /* Reset IRQ value */ 1007 snet->cfg_irq = -1; 1008 1009 ret = snet_open_vf_bar(pdev, snet); 1010 if (ret) 1011 goto put_device; 1012 1013 /* Create a VirtIO config pointer */ 1014 snet->cfg->virtio_cfg = snet->bar + snet->psnet->cfg.virtio_cfg_off; 1015 1016 pci_set_master(pdev); 1017 pci_set_drvdata(pdev, snet); 1018 1019 ret = snet_build_vqs(snet); 1020 if (ret) 1021 goto put_device; 1022 1023 /* Reserve IRQ indexes, 1024 * The IRQs may be requested and freed multiple times, 1025 * but the indexes won't change. 1026 */ 1027 snet_reserve_irq_idx(pf_irqs ? pdev_pf : pdev, snet); 1028 1029 /*set DMA device*/ 1030 snet->vdpa.dma_dev = &pdev->dev; 1031 1032 /* Register VDPA device */ 1033 ret = vdpa_register_device(&snet->vdpa, snet->cfg->vq_num); 1034 if (ret) { 1035 SNET_ERR(pdev, "Failed to register vdpa device\n"); 1036 goto free_vqs; 1037 } 1038 1039 return 0; 1040 1041 free_vqs: 1042 snet_free_vqs(snet); 1043 put_device: 1044 put_device(&snet->vdpa.dev); 1045 free_irqs: 1046 if (!pf_irqs) 1047 pci_free_irq_vectors(pdev); 1048 return ret; 1049 } 1050 1051 static int snet_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1052 { 1053 if (pdev->is_virtfn) 1054 return snet_vdpa_probe_vf(pdev); 1055 else 1056 return snet_vdpa_probe_pf(pdev); 1057 } 1058 1059 static void snet_vdpa_remove_pf(struct pci_dev *pdev) 1060 { 1061 struct psnet *psnet = pci_get_drvdata(pdev); 1062 1063 pci_disable_sriov(pdev); 1064 /* If IRQs are allocated from the PF, we should free the IRQs */ 1065 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF)) 1066 pci_free_irq_vectors(pdev); 1067 1068 snet_free_cfg(&psnet->cfg); 1069 kfree(psnet); 1070 } 1071 1072 static void snet_vdpa_remove_vf(struct pci_dev *pdev) 1073 { 1074 struct snet *snet = pci_get_drvdata(pdev); 1075 struct psnet *psnet = snet->psnet; 1076 1077 vdpa_unregister_device(&snet->vdpa); 1078 snet_free_vqs(snet); 1079 /* If IRQs are allocated from the VF, we should free the IRQs */ 1080 if (!PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF)) 1081 pci_free_irq_vectors(pdev); 1082 } 1083 1084 static void snet_vdpa_remove(struct pci_dev *pdev) 1085 { 1086 if (pdev->is_virtfn) 1087 snet_vdpa_remove_vf(pdev); 1088 else 1089 snet_vdpa_remove_pf(pdev); 1090 } 1091 1092 static struct pci_device_id snet_driver_pci_ids[] = { 1093 { PCI_DEVICE_SUB(PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID, 1094 PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID) }, 1095 { 0 }, 1096 }; 1097 1098 MODULE_DEVICE_TABLE(pci, snet_driver_pci_ids); 1099 1100 static struct pci_driver snet_vdpa_driver = { 1101 .name = "snet-vdpa-driver", 1102 .id_table = snet_driver_pci_ids, 1103 .probe = snet_vdpa_probe, 1104 .remove = snet_vdpa_remove, 1105 }; 1106 1107 module_pci_driver(snet_vdpa_driver); 1108 1109 MODULE_AUTHOR("Alvaro Karsz <alvaro.karsz@solid-run.com>"); 1110 MODULE_DESCRIPTION("SolidRun vDPA driver"); 1111 MODULE_LICENSE("GPL v2"); 1112