1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Endpoint Function Driver to implement Non-Transparent Bridge functionality 4 * Between PCI RC and EP 5 * 6 * Copyright (C) 2020 Texas Instruments 7 * Copyright (C) 2022 NXP 8 * 9 * Based on pci-epf-ntb.c 10 * Author: Frank Li <Frank.Li@nxp.com> 11 * Author: Kishon Vijay Abraham I <kishon@ti.com> 12 */ 13 14 /** 15 * +------------+ +---------------------------------------+ 16 * | | | | 17 * +------------+ | +--------------+ 18 * | NTB | | | NTB | 19 * | NetDev | | | NetDev | 20 * +------------+ | +--------------+ 21 * | NTB | | | NTB | 22 * | Transfer | | | Transfer | 23 * +------------+ | +--------------+ 24 * | | | | | 25 * | PCI NTB | | | | 26 * | EPF | | | | 27 * | Driver | | | PCI Virtual | 28 * | | +---------------+ | NTB Driver | 29 * | | | PCI EP NTB |<------>| | 30 * | | | FN Driver | | | 31 * +------------+ +---------------+ +--------------+ 32 * | | | | | | 33 * | PCI Bus | <-----> | PCI EP Bus | | Virtual PCI | 34 * | | PCI | | | Bus | 35 * +------------+ +---------------+--------+--------------+ 36 * PCIe Root Port PCI EP 37 */ 38 39 #include <linux/delay.h> 40 #include <linux/io.h> 41 #include <linux/module.h> 42 #include <linux/slab.h> 43 44 #include <linux/pci-epc.h> 45 #include <linux/pci-epf.h> 46 #include <linux/ntb.h> 47 48 static struct workqueue_struct *kpcintb_workqueue; 49 50 #define COMMAND_CONFIGURE_DOORBELL 1 51 #define COMMAND_TEARDOWN_DOORBELL 2 52 #define COMMAND_CONFIGURE_MW 3 53 #define COMMAND_TEARDOWN_MW 4 54 #define COMMAND_LINK_UP 5 55 #define COMMAND_LINK_DOWN 6 56 57 #define COMMAND_STATUS_OK 1 58 #define COMMAND_STATUS_ERROR 2 59 60 #define LINK_STATUS_UP BIT(0) 61 62 #define SPAD_COUNT 64 63 #define DB_COUNT 4 64 #define NTB_MW_OFFSET 2 65 #define DB_COUNT_MASK GENMASK(15, 0) 66 #define MSIX_ENABLE BIT(16) 67 #define MAX_DB_COUNT 32 68 #define MAX_MW 4 69 70 enum epf_ntb_bar { 71 BAR_CONFIG, 72 BAR_DB, 73 BAR_MW0, 74 BAR_MW1, 75 BAR_MW2, 76 }; 77 78 /* 79 * +--------------------------------------------------+ Base 80 * | | 81 * | | 82 * | | 83 * | Common Control Register | 84 * | | 85 * | | 86 * | | 87 * +-----------------------+--------------------------+ Base+span_offset 88 * | | | 89 * | Peer Span Space | Span Space | 90 * | | | 91 * | | | 92 * +-----------------------+--------------------------+ Base+span_offset 93 * | | | +span_count * 4 94 * | | | 95 * | Span Space | Peer Span Space | 96 * | | | 97 * +-----------------------+--------------------------+ 98 * Virtual PCI PCIe Endpoint 99 * NTB Driver NTB Driver 100 */ 101 struct epf_ntb_ctrl { 102 u32 command; 103 u32 argument; 104 u16 command_status; 105 u16 link_status; 106 u32 topology; 107 u64 addr; 108 u64 size; 109 u32 num_mws; 110 u32 reserved; 111 u32 spad_offset; 112 u32 spad_count; 113 u32 db_entry_size; 114 u32 db_data[MAX_DB_COUNT]; 115 u32 db_offset[MAX_DB_COUNT]; 116 } __packed; 117 118 struct epf_ntb { 119 struct ntb_dev ntb; 120 struct pci_epf *epf; 121 struct config_group group; 122 123 u32 num_mws; 124 u32 db_count; 125 u32 spad_count; 126 u64 mws_size[MAX_MW]; 127 u64 db; 128 u32 vbus_number; 129 u16 vntb_pid; 130 u16 vntb_vid; 131 132 bool linkup; 133 u32 spad_size; 134 135 enum pci_barno epf_ntb_bar[6]; 136 137 struct epf_ntb_ctrl *reg; 138 139 phys_addr_t epf_db_phy; 140 void __iomem *epf_db; 141 142 phys_addr_t vpci_mw_phy[MAX_MW]; 143 void __iomem *vpci_mw_addr[MAX_MW]; 144 145 struct delayed_work cmd_handler; 146 }; 147 148 #define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group) 149 #define ntb_ndev(__ntb) container_of(__ntb, struct epf_ntb, ntb) 150 151 static struct pci_epf_header epf_ntb_header = { 152 .vendorid = PCI_ANY_ID, 153 .deviceid = PCI_ANY_ID, 154 .baseclass_code = PCI_BASE_CLASS_MEMORY, 155 .interrupt_pin = PCI_INTERRUPT_INTA, 156 }; 157 158 /** 159 * epf_ntb_link_up() - Raise link_up interrupt to Virtual Host 160 * @ntb: NTB device that facilitates communication between HOST and VHOST 161 * @link_up: true or false indicating Link is UP or Down 162 * 163 * Once NTB function in HOST invoke ntb_link_enable(), 164 * this NTB function driver will trigger a link event to vhost. 165 */ 166 static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up) 167 { 168 if (link_up) 169 ntb->reg->link_status |= LINK_STATUS_UP; 170 else 171 ntb->reg->link_status &= ~LINK_STATUS_UP; 172 173 ntb_link_event(&ntb->ntb); 174 return 0; 175 } 176 177 /** 178 * epf_ntb_configure_mw() - Configure the Outbound Address Space for vhost 179 * to access the memory window of host 180 * @ntb: NTB device that facilitates communication between host and vhost 181 * @mw: Index of the memory window (either 0, 1, 2 or 3) 182 * 183 * EP Outbound Window 184 * +--------+ +-----------+ 185 * | | | | 186 * | | | | 187 * | | | | 188 * | | | | 189 * | | +-----------+ 190 * | Virtual| | Memory Win| 191 * | NTB | -----------> | | 192 * | Driver | | | 193 * | | +-----------+ 194 * | | | | 195 * | | | | 196 * +--------+ +-----------+ 197 * VHost PCI EP 198 */ 199 static int epf_ntb_configure_mw(struct epf_ntb *ntb, u32 mw) 200 { 201 phys_addr_t phys_addr; 202 u8 func_no, vfunc_no; 203 u64 addr, size; 204 int ret = 0; 205 206 phys_addr = ntb->vpci_mw_phy[mw]; 207 addr = ntb->reg->addr; 208 size = ntb->reg->size; 209 210 func_no = ntb->epf->func_no; 211 vfunc_no = ntb->epf->vfunc_no; 212 213 ret = pci_epc_map_addr(ntb->epf->epc, func_no, vfunc_no, phys_addr, addr, size); 214 if (ret) 215 dev_err(&ntb->epf->epc->dev, 216 "Failed to map memory window %d address\n", mw); 217 return ret; 218 } 219 220 /** 221 * epf_ntb_teardown_mw() - Teardown the configured OB ATU 222 * @ntb: NTB device that facilitates communication between HOST and vHOST 223 * @mw: Index of the memory window (either 0, 1, 2 or 3) 224 * 225 * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using 226 * pci_epc_unmap_addr() 227 */ 228 static void epf_ntb_teardown_mw(struct epf_ntb *ntb, u32 mw) 229 { 230 pci_epc_unmap_addr(ntb->epf->epc, 231 ntb->epf->func_no, 232 ntb->epf->vfunc_no, 233 ntb->vpci_mw_phy[mw]); 234 } 235 236 /** 237 * epf_ntb_cmd_handler() - Handle commands provided by the NTB Host 238 * @work: work_struct for the epf_ntb_epc 239 * 240 * Workqueue function that gets invoked for the two epf_ntb_epc 241 * periodically (once every 5ms) to see if it has received any commands 242 * from NTB host. The host can send commands to configure doorbell or 243 * configure memory window or to update link status. 244 */ 245 static void epf_ntb_cmd_handler(struct work_struct *work) 246 { 247 struct epf_ntb_ctrl *ctrl; 248 u32 command, argument; 249 struct epf_ntb *ntb; 250 struct device *dev; 251 int ret; 252 int i; 253 254 ntb = container_of(work, struct epf_ntb, cmd_handler.work); 255 256 for (i = 1; i < ntb->db_count; i++) { 257 if (readl(ntb->epf_db + i * 4)) { 258 if (readl(ntb->epf_db + i * 4)) 259 ntb->db |= 1 << (i - 1); 260 261 ntb_db_event(&ntb->ntb, i); 262 writel(0, ntb->epf_db + i * 4); 263 } 264 } 265 266 ctrl = ntb->reg; 267 command = ctrl->command; 268 if (!command) 269 goto reset_handler; 270 argument = ctrl->argument; 271 272 ctrl->command = 0; 273 ctrl->argument = 0; 274 275 ctrl = ntb->reg; 276 dev = &ntb->epf->dev; 277 278 switch (command) { 279 case COMMAND_CONFIGURE_DOORBELL: 280 ctrl->command_status = COMMAND_STATUS_OK; 281 break; 282 case COMMAND_TEARDOWN_DOORBELL: 283 ctrl->command_status = COMMAND_STATUS_OK; 284 break; 285 case COMMAND_CONFIGURE_MW: 286 ret = epf_ntb_configure_mw(ntb, argument); 287 if (ret < 0) 288 ctrl->command_status = COMMAND_STATUS_ERROR; 289 else 290 ctrl->command_status = COMMAND_STATUS_OK; 291 break; 292 case COMMAND_TEARDOWN_MW: 293 epf_ntb_teardown_mw(ntb, argument); 294 ctrl->command_status = COMMAND_STATUS_OK; 295 break; 296 case COMMAND_LINK_UP: 297 ntb->linkup = true; 298 ret = epf_ntb_link_up(ntb, true); 299 if (ret < 0) 300 ctrl->command_status = COMMAND_STATUS_ERROR; 301 else 302 ctrl->command_status = COMMAND_STATUS_OK; 303 goto reset_handler; 304 case COMMAND_LINK_DOWN: 305 ntb->linkup = false; 306 ret = epf_ntb_link_up(ntb, false); 307 if (ret < 0) 308 ctrl->command_status = COMMAND_STATUS_ERROR; 309 else 310 ctrl->command_status = COMMAND_STATUS_OK; 311 break; 312 default: 313 dev_err(dev, "UNKNOWN command: %d\n", command); 314 break; 315 } 316 317 reset_handler: 318 queue_delayed_work(kpcintb_workqueue, &ntb->cmd_handler, 319 msecs_to_jiffies(5)); 320 } 321 322 /** 323 * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR 324 * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound 325 * address. 326 * 327 * Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and 328 * self scratchpad region (removes inbound ATU configuration). While BAR0 is 329 * the default self scratchpad BAR, an NTB could have other BARs for self 330 * scratchpad (because of reserved BARs). This function can get the exact BAR 331 * used for self scratchpad from epf_ntb_bar[BAR_CONFIG]. 332 * 333 * Please note the self scratchpad region and config region is combined to 334 * a single region and mapped using the same BAR. Also note HOST2's peer 335 * scratchpad is HOST1's self scratchpad. 336 */ 337 static void epf_ntb_config_sspad_bar_clear(struct epf_ntb *ntb) 338 { 339 struct pci_epf_bar *epf_bar; 340 enum pci_barno barno; 341 342 barno = ntb->epf_ntb_bar[BAR_CONFIG]; 343 epf_bar = &ntb->epf->bar[barno]; 344 345 pci_epc_clear_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar); 346 } 347 348 /** 349 * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR 350 * @ntb: NTB device that facilitates communication between HOST and vHOST 351 * 352 * Map BAR0 of EP CONTROLLER 1 which contains the HOST1's config and 353 * self scratchpad region. 354 * 355 * Please note the self scratchpad region and config region is combined to 356 * a single region and mapped using the same BAR. 357 */ 358 static int epf_ntb_config_sspad_bar_set(struct epf_ntb *ntb) 359 { 360 struct pci_epf_bar *epf_bar; 361 enum pci_barno barno; 362 u8 func_no, vfunc_no; 363 struct device *dev; 364 int ret; 365 366 dev = &ntb->epf->dev; 367 func_no = ntb->epf->func_no; 368 vfunc_no = ntb->epf->vfunc_no; 369 barno = ntb->epf_ntb_bar[BAR_CONFIG]; 370 epf_bar = &ntb->epf->bar[barno]; 371 372 ret = pci_epc_set_bar(ntb->epf->epc, func_no, vfunc_no, epf_bar); 373 if (ret) { 374 dev_err(dev, "inft: Config/Status/SPAD BAR set failed\n"); 375 return ret; 376 } 377 return 0; 378 } 379 380 /** 381 * epf_ntb_config_spad_bar_free() - Free the physical memory associated with 382 * config + scratchpad region 383 * @ntb: NTB device that facilitates communication between HOST and vHOST 384 */ 385 static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb) 386 { 387 enum pci_barno barno; 388 389 barno = ntb->epf_ntb_bar[BAR_CONFIG]; 390 pci_epf_free_space(ntb->epf, ntb->reg, barno, 0); 391 } 392 393 /** 394 * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad 395 * region 396 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 397 * 398 * Allocate the Local Memory mentioned in the above diagram. The size of 399 * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION 400 * is obtained from "spad-count" configfs entry. 401 */ 402 static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb) 403 { 404 size_t align; 405 enum pci_barno barno; 406 struct epf_ntb_ctrl *ctrl; 407 u32 spad_size, ctrl_size; 408 u64 size; 409 struct pci_epf *epf = ntb->epf; 410 struct device *dev = &epf->dev; 411 u32 spad_count; 412 void *base; 413 int i; 414 const struct pci_epc_features *epc_features = pci_epc_get_features(epf->epc, 415 epf->func_no, 416 epf->vfunc_no); 417 barno = ntb->epf_ntb_bar[BAR_CONFIG]; 418 size = epc_features->bar_fixed_size[barno]; 419 align = epc_features->align; 420 421 if ((!IS_ALIGNED(size, align))) 422 return -EINVAL; 423 424 spad_count = ntb->spad_count; 425 426 ctrl_size = sizeof(struct epf_ntb_ctrl); 427 spad_size = 2 * spad_count * 4; 428 429 if (!align) { 430 ctrl_size = roundup_pow_of_two(ctrl_size); 431 spad_size = roundup_pow_of_two(spad_size); 432 } else { 433 ctrl_size = ALIGN(ctrl_size, align); 434 spad_size = ALIGN(spad_size, align); 435 } 436 437 if (!size) 438 size = ctrl_size + spad_size; 439 else if (size < ctrl_size + spad_size) 440 return -EINVAL; 441 442 base = pci_epf_alloc_space(epf, size, barno, align, 0); 443 if (!base) { 444 dev_err(dev, "Config/Status/SPAD alloc region fail\n"); 445 return -ENOMEM; 446 } 447 448 ntb->reg = base; 449 450 ctrl = ntb->reg; 451 ctrl->spad_offset = ctrl_size; 452 453 ctrl->spad_count = spad_count; 454 ctrl->num_mws = ntb->num_mws; 455 ntb->spad_size = spad_size; 456 457 ctrl->db_entry_size = 4; 458 459 for (i = 0; i < ntb->db_count; i++) { 460 ntb->reg->db_data[i] = 1 + i; 461 ntb->reg->db_offset[i] = 0; 462 } 463 464 return 0; 465 } 466 467 /** 468 * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capaiblity 469 * @ntb: NTB device that facilitates communication between HOST and vHOST 470 * 471 * Configure MSI/MSI-X capability for each interface with number of 472 * interrupts equal to "db_count" configfs entry. 473 */ 474 static int epf_ntb_configure_interrupt(struct epf_ntb *ntb) 475 { 476 const struct pci_epc_features *epc_features; 477 struct device *dev; 478 u32 db_count; 479 int ret; 480 481 dev = &ntb->epf->dev; 482 483 epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no); 484 485 if (!(epc_features->msix_capable || epc_features->msi_capable)) { 486 dev_err(dev, "MSI or MSI-X is required for doorbell\n"); 487 return -EINVAL; 488 } 489 490 db_count = ntb->db_count; 491 if (db_count > MAX_DB_COUNT) { 492 dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT); 493 return -EINVAL; 494 } 495 496 ntb->db_count = db_count; 497 498 if (epc_features->msi_capable) { 499 ret = pci_epc_set_msi(ntb->epf->epc, 500 ntb->epf->func_no, 501 ntb->epf->vfunc_no, 502 16); 503 if (ret) { 504 dev_err(dev, "MSI configuration failed\n"); 505 return ret; 506 } 507 } 508 509 return 0; 510 } 511 512 /** 513 * epf_ntb_db_bar_init() - Configure Doorbell window BARs 514 * @ntb: NTB device that facilitates communication between HOST and vHOST 515 */ 516 static int epf_ntb_db_bar_init(struct epf_ntb *ntb) 517 { 518 const struct pci_epc_features *epc_features; 519 u32 align; 520 struct device *dev = &ntb->epf->dev; 521 int ret; 522 struct pci_epf_bar *epf_bar; 523 void __iomem *mw_addr; 524 enum pci_barno barno; 525 size_t size = 4 * ntb->db_count; 526 527 epc_features = pci_epc_get_features(ntb->epf->epc, 528 ntb->epf->func_no, 529 ntb->epf->vfunc_no); 530 align = epc_features->align; 531 532 if (size < 128) 533 size = 128; 534 535 if (align) 536 size = ALIGN(size, align); 537 else 538 size = roundup_pow_of_two(size); 539 540 barno = ntb->epf_ntb_bar[BAR_DB]; 541 542 mw_addr = pci_epf_alloc_space(ntb->epf, size, barno, align, 0); 543 if (!mw_addr) { 544 dev_err(dev, "Failed to allocate OB address\n"); 545 return -ENOMEM; 546 } 547 548 ntb->epf_db = mw_addr; 549 550 epf_bar = &ntb->epf->bar[barno]; 551 552 ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar); 553 if (ret) { 554 dev_err(dev, "Doorbell BAR set failed\n"); 555 goto err_alloc_peer_mem; 556 } 557 return ret; 558 559 err_alloc_peer_mem: 560 pci_epc_mem_free_addr(ntb->epf->epc, epf_bar->phys_addr, mw_addr, epf_bar->size); 561 return -1; 562 } 563 564 static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws); 565 566 /** 567 * epf_ntb_db_bar_clear() - Clear doorbell BAR and free memory 568 * allocated in peer's outbound address space 569 * @ntb: NTB device that facilitates communication between HOST and vHOST 570 */ 571 static void epf_ntb_db_bar_clear(struct epf_ntb *ntb) 572 { 573 enum pci_barno barno; 574 575 barno = ntb->epf_ntb_bar[BAR_DB]; 576 pci_epf_free_space(ntb->epf, ntb->epf_db, barno, 0); 577 pci_epc_clear_bar(ntb->epf->epc, 578 ntb->epf->func_no, 579 ntb->epf->vfunc_no, 580 &ntb->epf->bar[barno]); 581 } 582 583 /** 584 * epf_ntb_mw_bar_init() - Configure Memory window BARs 585 * @ntb: NTB device that facilitates communication between HOST and vHOST 586 * 587 */ 588 static int epf_ntb_mw_bar_init(struct epf_ntb *ntb) 589 { 590 int ret = 0; 591 int i; 592 u64 size; 593 enum pci_barno barno; 594 struct device *dev = &ntb->epf->dev; 595 596 for (i = 0; i < ntb->num_mws; i++) { 597 size = ntb->mws_size[i]; 598 barno = ntb->epf_ntb_bar[BAR_MW0 + i]; 599 600 ntb->epf->bar[barno].barno = barno; 601 ntb->epf->bar[barno].size = size; 602 ntb->epf->bar[barno].addr = NULL; 603 ntb->epf->bar[barno].phys_addr = 0; 604 ntb->epf->bar[barno].flags |= upper_32_bits(size) ? 605 PCI_BASE_ADDRESS_MEM_TYPE_64 : 606 PCI_BASE_ADDRESS_MEM_TYPE_32; 607 608 ret = pci_epc_set_bar(ntb->epf->epc, 609 ntb->epf->func_no, 610 ntb->epf->vfunc_no, 611 &ntb->epf->bar[barno]); 612 if (ret) { 613 dev_err(dev, "MW set failed\n"); 614 goto err_alloc_mem; 615 } 616 617 /* Allocate EPC outbound memory windows to vpci vntb device */ 618 ntb->vpci_mw_addr[i] = pci_epc_mem_alloc_addr(ntb->epf->epc, 619 &ntb->vpci_mw_phy[i], 620 size); 621 if (!ntb->vpci_mw_addr[i]) { 622 ret = -ENOMEM; 623 dev_err(dev, "Failed to allocate source address\n"); 624 goto err_set_bar; 625 } 626 } 627 628 return ret; 629 630 err_set_bar: 631 pci_epc_clear_bar(ntb->epf->epc, 632 ntb->epf->func_no, 633 ntb->epf->vfunc_no, 634 &ntb->epf->bar[barno]); 635 err_alloc_mem: 636 epf_ntb_mw_bar_clear(ntb, i); 637 return ret; 638 } 639 640 /** 641 * epf_ntb_mw_bar_clear() - Clear Memory window BARs 642 * @ntb: NTB device that facilitates communication between HOST and vHOST 643 */ 644 static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws) 645 { 646 enum pci_barno barno; 647 int i; 648 649 for (i = 0; i < num_mws; i++) { 650 barno = ntb->epf_ntb_bar[BAR_MW0 + i]; 651 pci_epc_clear_bar(ntb->epf->epc, 652 ntb->epf->func_no, 653 ntb->epf->vfunc_no, 654 &ntb->epf->bar[barno]); 655 656 pci_epc_mem_free_addr(ntb->epf->epc, 657 ntb->vpci_mw_phy[i], 658 ntb->vpci_mw_addr[i], 659 ntb->mws_size[i]); 660 } 661 } 662 663 /** 664 * epf_ntb_epc_destroy() - Cleanup NTB EPC interface 665 * @ntb: NTB device that facilitates communication between HOST and vHOST 666 * 667 * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces 668 */ 669 static void epf_ntb_epc_destroy(struct epf_ntb *ntb) 670 { 671 pci_epc_remove_epf(ntb->epf->epc, ntb->epf, 0); 672 pci_epc_put(ntb->epf->epc); 673 } 674 675 /** 676 * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB 677 * constructs (scratchpad region, doorbell, memorywindow) 678 * @ntb: NTB device that facilitates communication between HOST and vHOST 679 */ 680 static int epf_ntb_init_epc_bar(struct epf_ntb *ntb) 681 { 682 const struct pci_epc_features *epc_features; 683 enum pci_barno barno; 684 enum epf_ntb_bar bar; 685 struct device *dev; 686 u32 num_mws; 687 int i; 688 689 barno = BAR_0; 690 num_mws = ntb->num_mws; 691 dev = &ntb->epf->dev; 692 epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no); 693 694 /* These are required BARs which are mandatory for NTB functionality */ 695 for (bar = BAR_CONFIG; bar <= BAR_MW0; bar++, barno++) { 696 barno = pci_epc_get_next_free_bar(epc_features, barno); 697 if (barno < 0) { 698 dev_err(dev, "Fail to get NTB function BAR\n"); 699 return barno; 700 } 701 ntb->epf_ntb_bar[bar] = barno; 702 } 703 704 /* These are optional BARs which don't impact NTB functionality */ 705 for (bar = BAR_MW1, i = 1; i < num_mws; bar++, barno++, i++) { 706 barno = pci_epc_get_next_free_bar(epc_features, barno); 707 if (barno < 0) { 708 ntb->num_mws = i; 709 dev_dbg(dev, "BAR not available for > MW%d\n", i + 1); 710 } 711 ntb->epf_ntb_bar[bar] = barno; 712 } 713 714 return 0; 715 } 716 717 /** 718 * epf_ntb_epc_init() - Initialize NTB interface 719 * @ntb: NTB device that facilitates communication between HOST and vHOST2 720 * 721 * Wrapper to initialize a particular EPC interface and start the workqueue 722 * to check for commands from host. This function will write to the 723 * EP controller HW for configuring it. 724 */ 725 static int epf_ntb_epc_init(struct epf_ntb *ntb) 726 { 727 u8 func_no, vfunc_no; 728 struct pci_epc *epc; 729 struct pci_epf *epf; 730 struct device *dev; 731 int ret; 732 733 epf = ntb->epf; 734 dev = &epf->dev; 735 epc = epf->epc; 736 func_no = ntb->epf->func_no; 737 vfunc_no = ntb->epf->vfunc_no; 738 739 ret = epf_ntb_config_sspad_bar_set(ntb); 740 if (ret) { 741 dev_err(dev, "Config/self SPAD BAR init failed"); 742 return ret; 743 } 744 745 ret = epf_ntb_configure_interrupt(ntb); 746 if (ret) { 747 dev_err(dev, "Interrupt configuration failed\n"); 748 goto err_config_interrupt; 749 } 750 751 ret = epf_ntb_db_bar_init(ntb); 752 if (ret) { 753 dev_err(dev, "DB BAR init failed\n"); 754 goto err_db_bar_init; 755 } 756 757 ret = epf_ntb_mw_bar_init(ntb); 758 if (ret) { 759 dev_err(dev, "MW BAR init failed\n"); 760 goto err_mw_bar_init; 761 } 762 763 if (vfunc_no <= 1) { 764 ret = pci_epc_write_header(epc, func_no, vfunc_no, epf->header); 765 if (ret) { 766 dev_err(dev, "Configuration header write failed\n"); 767 goto err_write_header; 768 } 769 } 770 771 INIT_DELAYED_WORK(&ntb->cmd_handler, epf_ntb_cmd_handler); 772 queue_work(kpcintb_workqueue, &ntb->cmd_handler.work); 773 774 return 0; 775 776 err_write_header: 777 epf_ntb_mw_bar_clear(ntb, ntb->num_mws); 778 err_mw_bar_init: 779 epf_ntb_db_bar_clear(ntb); 780 err_db_bar_init: 781 err_config_interrupt: 782 epf_ntb_config_sspad_bar_clear(ntb); 783 784 return ret; 785 } 786 787 788 /** 789 * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces 790 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 791 * 792 * Wrapper to cleanup all NTB interfaces. 793 */ 794 static void epf_ntb_epc_cleanup(struct epf_ntb *ntb) 795 { 796 epf_ntb_db_bar_clear(ntb); 797 epf_ntb_mw_bar_clear(ntb, ntb->num_mws); 798 } 799 800 #define EPF_NTB_R(_name) \ 801 static ssize_t epf_ntb_##_name##_show(struct config_item *item, \ 802 char *page) \ 803 { \ 804 struct config_group *group = to_config_group(item); \ 805 struct epf_ntb *ntb = to_epf_ntb(group); \ 806 \ 807 return sprintf(page, "%d\n", ntb->_name); \ 808 } 809 810 #define EPF_NTB_W(_name) \ 811 static ssize_t epf_ntb_##_name##_store(struct config_item *item, \ 812 const char *page, size_t len) \ 813 { \ 814 struct config_group *group = to_config_group(item); \ 815 struct epf_ntb *ntb = to_epf_ntb(group); \ 816 u32 val; \ 817 int ret; \ 818 \ 819 ret = kstrtou32(page, 0, &val); \ 820 if (ret) \ 821 return ret; \ 822 \ 823 ntb->_name = val; \ 824 \ 825 return len; \ 826 } 827 828 #define EPF_NTB_MW_R(_name) \ 829 static ssize_t epf_ntb_##_name##_show(struct config_item *item, \ 830 char *page) \ 831 { \ 832 struct config_group *group = to_config_group(item); \ 833 struct epf_ntb *ntb = to_epf_ntb(group); \ 834 struct device *dev = &ntb->epf->dev; \ 835 int win_no; \ 836 \ 837 if (sscanf(#_name, "mw%d", &win_no) != 1) \ 838 return -EINVAL; \ 839 \ 840 if (win_no <= 0 || win_no > ntb->num_mws) { \ 841 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \ 842 return -EINVAL; \ 843 } \ 844 \ 845 return sprintf(page, "%lld\n", ntb->mws_size[win_no - 1]); \ 846 } 847 848 #define EPF_NTB_MW_W(_name) \ 849 static ssize_t epf_ntb_##_name##_store(struct config_item *item, \ 850 const char *page, size_t len) \ 851 { \ 852 struct config_group *group = to_config_group(item); \ 853 struct epf_ntb *ntb = to_epf_ntb(group); \ 854 struct device *dev = &ntb->epf->dev; \ 855 int win_no; \ 856 u64 val; \ 857 int ret; \ 858 \ 859 ret = kstrtou64(page, 0, &val); \ 860 if (ret) \ 861 return ret; \ 862 \ 863 if (sscanf(#_name, "mw%d", &win_no) != 1) \ 864 return -EINVAL; \ 865 \ 866 if (win_no <= 0 || win_no > ntb->num_mws) { \ 867 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \ 868 return -EINVAL; \ 869 } \ 870 \ 871 ntb->mws_size[win_no - 1] = val; \ 872 \ 873 return len; \ 874 } 875 876 static ssize_t epf_ntb_num_mws_store(struct config_item *item, 877 const char *page, size_t len) 878 { 879 struct config_group *group = to_config_group(item); 880 struct epf_ntb *ntb = to_epf_ntb(group); 881 u32 val; 882 int ret; 883 884 ret = kstrtou32(page, 0, &val); 885 if (ret) 886 return ret; 887 888 if (val > MAX_MW) 889 return -EINVAL; 890 891 ntb->num_mws = val; 892 893 return len; 894 } 895 896 EPF_NTB_R(spad_count) 897 EPF_NTB_W(spad_count) 898 EPF_NTB_R(db_count) 899 EPF_NTB_W(db_count) 900 EPF_NTB_R(num_mws) 901 EPF_NTB_R(vbus_number) 902 EPF_NTB_W(vbus_number) 903 EPF_NTB_R(vntb_pid) 904 EPF_NTB_W(vntb_pid) 905 EPF_NTB_R(vntb_vid) 906 EPF_NTB_W(vntb_vid) 907 EPF_NTB_MW_R(mw1) 908 EPF_NTB_MW_W(mw1) 909 EPF_NTB_MW_R(mw2) 910 EPF_NTB_MW_W(mw2) 911 EPF_NTB_MW_R(mw3) 912 EPF_NTB_MW_W(mw3) 913 EPF_NTB_MW_R(mw4) 914 EPF_NTB_MW_W(mw4) 915 916 CONFIGFS_ATTR(epf_ntb_, spad_count); 917 CONFIGFS_ATTR(epf_ntb_, db_count); 918 CONFIGFS_ATTR(epf_ntb_, num_mws); 919 CONFIGFS_ATTR(epf_ntb_, mw1); 920 CONFIGFS_ATTR(epf_ntb_, mw2); 921 CONFIGFS_ATTR(epf_ntb_, mw3); 922 CONFIGFS_ATTR(epf_ntb_, mw4); 923 CONFIGFS_ATTR(epf_ntb_, vbus_number); 924 CONFIGFS_ATTR(epf_ntb_, vntb_pid); 925 CONFIGFS_ATTR(epf_ntb_, vntb_vid); 926 927 static struct configfs_attribute *epf_ntb_attrs[] = { 928 &epf_ntb_attr_spad_count, 929 &epf_ntb_attr_db_count, 930 &epf_ntb_attr_num_mws, 931 &epf_ntb_attr_mw1, 932 &epf_ntb_attr_mw2, 933 &epf_ntb_attr_mw3, 934 &epf_ntb_attr_mw4, 935 &epf_ntb_attr_vbus_number, 936 &epf_ntb_attr_vntb_pid, 937 &epf_ntb_attr_vntb_vid, 938 NULL, 939 }; 940 941 static const struct config_item_type ntb_group_type = { 942 .ct_attrs = epf_ntb_attrs, 943 .ct_owner = THIS_MODULE, 944 }; 945 946 /** 947 * epf_ntb_add_cfs() - Add configfs directory specific to NTB 948 * @epf: NTB endpoint function device 949 * @group: A pointer to the config_group structure referencing a group of 950 * config_items of a specific type that belong to a specific sub-system. 951 * 952 * Add configfs directory specific to NTB. This directory will hold 953 * NTB specific properties like db_count, spad_count, num_mws etc., 954 */ 955 static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf, 956 struct config_group *group) 957 { 958 struct epf_ntb *ntb = epf_get_drvdata(epf); 959 struct config_group *ntb_group = &ntb->group; 960 struct device *dev = &epf->dev; 961 962 config_group_init_type_name(ntb_group, dev_name(dev), &ntb_group_type); 963 964 return ntb_group; 965 } 966 967 /*==== virtual PCI bus driver, which only load virtual NTB PCI driver ====*/ 968 969 static u32 pci_space[] = { 970 0xffffffff, /*DeviceID, Vendor ID*/ 971 0, /*Status, Command*/ 972 0xffffffff, /*Class code, subclass, prog if, revision id*/ 973 0x40, /*bist, header type, latency Timer, cache line size*/ 974 0, /*BAR 0*/ 975 0, /*BAR 1*/ 976 0, /*BAR 2*/ 977 0, /*BAR 3*/ 978 0, /*BAR 4*/ 979 0, /*BAR 5*/ 980 0, /*Cardbus cis point*/ 981 0, /*Subsystem ID Subystem vendor id*/ 982 0, /*ROM Base Address*/ 983 0, /*Reserved, Cap. Point*/ 984 0, /*Reserved,*/ 985 0, /*Max Lat, Min Gnt, interrupt pin, interrupt line*/ 986 }; 987 988 static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val) 989 { 990 if (devfn == 0) { 991 memcpy(val, ((u8 *)pci_space) + where, size); 992 return PCIBIOS_SUCCESSFUL; 993 } 994 return PCIBIOS_DEVICE_NOT_FOUND; 995 } 996 997 static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val) 998 { 999 return 0; 1000 } 1001 1002 static struct pci_ops vpci_ops = { 1003 .read = pci_read, 1004 .write = pci_write, 1005 }; 1006 1007 static int vpci_scan_bus(void *sysdata) 1008 { 1009 struct pci_bus *vpci_bus; 1010 struct epf_ntb *ndev = sysdata; 1011 1012 vpci_bus = pci_scan_bus(ndev->vbus_number, &vpci_ops, sysdata); 1013 if (vpci_bus) 1014 pr_err("create pci bus\n"); 1015 1016 pci_bus_add_devices(vpci_bus); 1017 1018 return 0; 1019 } 1020 1021 /*==================== Virtual PCIe NTB driver ==========================*/ 1022 1023 static int vntb_epf_mw_count(struct ntb_dev *ntb, int pidx) 1024 { 1025 struct epf_ntb *ndev = ntb_ndev(ntb); 1026 1027 return ndev->num_mws; 1028 } 1029 1030 static int vntb_epf_spad_count(struct ntb_dev *ntb) 1031 { 1032 return ntb_ndev(ntb)->spad_count; 1033 } 1034 1035 static int vntb_epf_peer_mw_count(struct ntb_dev *ntb) 1036 { 1037 return ntb_ndev(ntb)->num_mws; 1038 } 1039 1040 static u64 vntb_epf_db_valid_mask(struct ntb_dev *ntb) 1041 { 1042 return BIT_ULL(ntb_ndev(ntb)->db_count) - 1; 1043 } 1044 1045 static int vntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits) 1046 { 1047 return 0; 1048 } 1049 1050 static int vntb_epf_mw_set_trans(struct ntb_dev *ndev, int pidx, int idx, 1051 dma_addr_t addr, resource_size_t size) 1052 { 1053 struct epf_ntb *ntb = ntb_ndev(ndev); 1054 struct pci_epf_bar *epf_bar; 1055 enum pci_barno barno; 1056 int ret; 1057 struct device *dev; 1058 1059 dev = &ntb->ntb.dev; 1060 barno = ntb->epf_ntb_bar[BAR_MW0 + idx]; 1061 epf_bar = &ntb->epf->bar[barno]; 1062 epf_bar->phys_addr = addr; 1063 epf_bar->barno = barno; 1064 epf_bar->size = size; 1065 1066 ret = pci_epc_set_bar(ntb->epf->epc, 0, 0, epf_bar); 1067 if (ret) { 1068 dev_err(dev, "failure set mw trans\n"); 1069 return ret; 1070 } 1071 return 0; 1072 } 1073 1074 static int vntb_epf_mw_clear_trans(struct ntb_dev *ntb, int pidx, int idx) 1075 { 1076 return 0; 1077 } 1078 1079 static int vntb_epf_peer_mw_get_addr(struct ntb_dev *ndev, int idx, 1080 phys_addr_t *base, resource_size_t *size) 1081 { 1082 1083 struct epf_ntb *ntb = ntb_ndev(ndev); 1084 1085 if (base) 1086 *base = ntb->vpci_mw_phy[idx]; 1087 1088 if (size) 1089 *size = ntb->mws_size[idx]; 1090 1091 return 0; 1092 } 1093 1094 static int vntb_epf_link_enable(struct ntb_dev *ntb, 1095 enum ntb_speed max_speed, 1096 enum ntb_width max_width) 1097 { 1098 return 0; 1099 } 1100 1101 static u32 vntb_epf_spad_read(struct ntb_dev *ndev, int idx) 1102 { 1103 struct epf_ntb *ntb = ntb_ndev(ndev); 1104 int off = ntb->reg->spad_offset, ct = ntb->reg->spad_count * 4; 1105 u32 val; 1106 void __iomem *base = ntb->reg; 1107 1108 val = readl(base + off + ct + idx * 4); 1109 return val; 1110 } 1111 1112 static int vntb_epf_spad_write(struct ntb_dev *ndev, int idx, u32 val) 1113 { 1114 struct epf_ntb *ntb = ntb_ndev(ndev); 1115 struct epf_ntb_ctrl *ctrl = ntb->reg; 1116 int off = ctrl->spad_offset, ct = ctrl->spad_count * 4; 1117 void __iomem *base = ntb->reg; 1118 1119 writel(val, base + off + ct + idx * 4); 1120 return 0; 1121 } 1122 1123 static u32 vntb_epf_peer_spad_read(struct ntb_dev *ndev, int pidx, int idx) 1124 { 1125 struct epf_ntb *ntb = ntb_ndev(ndev); 1126 struct epf_ntb_ctrl *ctrl = ntb->reg; 1127 int off = ctrl->spad_offset; 1128 void __iomem *base = ntb->reg; 1129 u32 val; 1130 1131 val = readl(base + off + idx * 4); 1132 return val; 1133 } 1134 1135 static int vntb_epf_peer_spad_write(struct ntb_dev *ndev, int pidx, int idx, u32 val) 1136 { 1137 struct epf_ntb *ntb = ntb_ndev(ndev); 1138 struct epf_ntb_ctrl *ctrl = ntb->reg; 1139 int off = ctrl->spad_offset; 1140 void __iomem *base = ntb->reg; 1141 1142 writel(val, base + off + idx * 4); 1143 return 0; 1144 } 1145 1146 static int vntb_epf_peer_db_set(struct ntb_dev *ndev, u64 db_bits) 1147 { 1148 u32 interrupt_num = ffs(db_bits) + 1; 1149 struct epf_ntb *ntb = ntb_ndev(ndev); 1150 u8 func_no, vfunc_no; 1151 int ret; 1152 1153 func_no = ntb->epf->func_no; 1154 vfunc_no = ntb->epf->vfunc_no; 1155 1156 ret = pci_epc_raise_irq(ntb->epf->epc, 1157 func_no, 1158 vfunc_no, 1159 PCI_EPC_IRQ_MSI, 1160 interrupt_num + 1); 1161 if (ret) 1162 dev_err(&ntb->ntb.dev, "Failed to raise IRQ\n"); 1163 1164 return ret; 1165 } 1166 1167 static u64 vntb_epf_db_read(struct ntb_dev *ndev) 1168 { 1169 struct epf_ntb *ntb = ntb_ndev(ndev); 1170 1171 return ntb->db; 1172 } 1173 1174 static int vntb_epf_mw_get_align(struct ntb_dev *ndev, int pidx, int idx, 1175 resource_size_t *addr_align, 1176 resource_size_t *size_align, 1177 resource_size_t *size_max) 1178 { 1179 struct epf_ntb *ntb = ntb_ndev(ndev); 1180 1181 if (addr_align) 1182 *addr_align = SZ_4K; 1183 1184 if (size_align) 1185 *size_align = 1; 1186 1187 if (size_max) 1188 *size_max = ntb->mws_size[idx]; 1189 1190 return 0; 1191 } 1192 1193 static u64 vntb_epf_link_is_up(struct ntb_dev *ndev, 1194 enum ntb_speed *speed, 1195 enum ntb_width *width) 1196 { 1197 struct epf_ntb *ntb = ntb_ndev(ndev); 1198 1199 return ntb->reg->link_status; 1200 } 1201 1202 static int vntb_epf_db_clear_mask(struct ntb_dev *ndev, u64 db_bits) 1203 { 1204 return 0; 1205 } 1206 1207 static int vntb_epf_db_clear(struct ntb_dev *ndev, u64 db_bits) 1208 { 1209 struct epf_ntb *ntb = ntb_ndev(ndev); 1210 1211 ntb->db &= ~db_bits; 1212 return 0; 1213 } 1214 1215 static int vntb_epf_link_disable(struct ntb_dev *ntb) 1216 { 1217 return 0; 1218 } 1219 1220 static const struct ntb_dev_ops vntb_epf_ops = { 1221 .mw_count = vntb_epf_mw_count, 1222 .spad_count = vntb_epf_spad_count, 1223 .peer_mw_count = vntb_epf_peer_mw_count, 1224 .db_valid_mask = vntb_epf_db_valid_mask, 1225 .db_set_mask = vntb_epf_db_set_mask, 1226 .mw_set_trans = vntb_epf_mw_set_trans, 1227 .mw_clear_trans = vntb_epf_mw_clear_trans, 1228 .peer_mw_get_addr = vntb_epf_peer_mw_get_addr, 1229 .link_enable = vntb_epf_link_enable, 1230 .spad_read = vntb_epf_spad_read, 1231 .spad_write = vntb_epf_spad_write, 1232 .peer_spad_read = vntb_epf_peer_spad_read, 1233 .peer_spad_write = vntb_epf_peer_spad_write, 1234 .peer_db_set = vntb_epf_peer_db_set, 1235 .db_read = vntb_epf_db_read, 1236 .mw_get_align = vntb_epf_mw_get_align, 1237 .link_is_up = vntb_epf_link_is_up, 1238 .db_clear_mask = vntb_epf_db_clear_mask, 1239 .db_clear = vntb_epf_db_clear, 1240 .link_disable = vntb_epf_link_disable, 1241 }; 1242 1243 static int pci_vntb_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1244 { 1245 int ret; 1246 struct epf_ntb *ndev = (struct epf_ntb *)pdev->sysdata; 1247 struct device *dev = &pdev->dev; 1248 1249 ndev->ntb.pdev = pdev; 1250 ndev->ntb.topo = NTB_TOPO_NONE; 1251 ndev->ntb.ops = &vntb_epf_ops; 1252 1253 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); 1254 if (ret) { 1255 dev_err(dev, "Cannot set DMA mask\n"); 1256 return -EINVAL; 1257 } 1258 1259 ret = ntb_register_device(&ndev->ntb); 1260 if (ret) { 1261 dev_err(dev, "Failed to register NTB device\n"); 1262 goto err_register_dev; 1263 } 1264 1265 dev_dbg(dev, "PCI Virtual NTB driver loaded\n"); 1266 return 0; 1267 1268 err_register_dev: 1269 return -EINVAL; 1270 } 1271 1272 static struct pci_device_id pci_vntb_table[] = { 1273 { 1274 PCI_DEVICE(0xffff, 0xffff), 1275 }, 1276 {}, 1277 }; 1278 1279 static struct pci_driver vntb_pci_driver = { 1280 .name = "pci-vntb", 1281 .id_table = pci_vntb_table, 1282 .probe = pci_vntb_probe, 1283 }; 1284 1285 /* ============ PCIe EPF Driver Bind ====================*/ 1286 1287 /** 1288 * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality 1289 * @epf: NTB endpoint function device 1290 * 1291 * Initialize both the endpoint controllers associated with NTB function device. 1292 * Invoked when a primary interface or secondary interface is bound to EPC 1293 * device. This function will succeed only when EPC is bound to both the 1294 * interfaces. 1295 */ 1296 static int epf_ntb_bind(struct pci_epf *epf) 1297 { 1298 struct epf_ntb *ntb = epf_get_drvdata(epf); 1299 struct device *dev = &epf->dev; 1300 int ret; 1301 1302 if (!epf->epc) { 1303 dev_dbg(dev, "PRIMARY EPC interface not yet bound\n"); 1304 return 0; 1305 } 1306 1307 ret = epf_ntb_init_epc_bar(ntb); 1308 if (ret) { 1309 dev_err(dev, "Failed to create NTB EPC\n"); 1310 goto err_bar_init; 1311 } 1312 1313 ret = epf_ntb_config_spad_bar_alloc(ntb); 1314 if (ret) { 1315 dev_err(dev, "Failed to allocate BAR memory\n"); 1316 goto err_bar_alloc; 1317 } 1318 1319 ret = epf_ntb_epc_init(ntb); 1320 if (ret) { 1321 dev_err(dev, "Failed to initialize EPC\n"); 1322 goto err_bar_alloc; 1323 } 1324 1325 epf_set_drvdata(epf, ntb); 1326 1327 pci_space[0] = (ntb->vntb_pid << 16) | ntb->vntb_vid; 1328 pci_vntb_table[0].vendor = ntb->vntb_vid; 1329 pci_vntb_table[0].device = ntb->vntb_pid; 1330 1331 ret = pci_register_driver(&vntb_pci_driver); 1332 if (ret) { 1333 dev_err(dev, "failure register vntb pci driver\n"); 1334 goto err_bar_alloc; 1335 } 1336 1337 vpci_scan_bus(ntb); 1338 1339 return 0; 1340 1341 err_bar_alloc: 1342 epf_ntb_config_spad_bar_free(ntb); 1343 1344 err_bar_init: 1345 epf_ntb_epc_destroy(ntb); 1346 1347 return ret; 1348 } 1349 1350 /** 1351 * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind() 1352 * @epf: NTB endpoint function device 1353 * 1354 * Cleanup the initialization from epf_ntb_bind() 1355 */ 1356 static void epf_ntb_unbind(struct pci_epf *epf) 1357 { 1358 struct epf_ntb *ntb = epf_get_drvdata(epf); 1359 1360 epf_ntb_epc_cleanup(ntb); 1361 epf_ntb_config_spad_bar_free(ntb); 1362 epf_ntb_epc_destroy(ntb); 1363 1364 pci_unregister_driver(&vntb_pci_driver); 1365 } 1366 1367 // EPF driver probe 1368 static struct pci_epf_ops epf_ntb_ops = { 1369 .bind = epf_ntb_bind, 1370 .unbind = epf_ntb_unbind, 1371 .add_cfs = epf_ntb_add_cfs, 1372 }; 1373 1374 /** 1375 * epf_ntb_probe() - Probe NTB function driver 1376 * @epf: NTB endpoint function device 1377 * 1378 * Probe NTB function driver when endpoint function bus detects a NTB 1379 * endpoint function. 1380 */ 1381 static int epf_ntb_probe(struct pci_epf *epf) 1382 { 1383 struct epf_ntb *ntb; 1384 struct device *dev; 1385 1386 dev = &epf->dev; 1387 1388 ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL); 1389 if (!ntb) 1390 return -ENOMEM; 1391 1392 epf->header = &epf_ntb_header; 1393 ntb->epf = epf; 1394 ntb->vbus_number = 0xff; 1395 epf_set_drvdata(epf, ntb); 1396 1397 dev_info(dev, "pci-ep epf driver loaded\n"); 1398 return 0; 1399 } 1400 1401 static const struct pci_epf_device_id epf_ntb_ids[] = { 1402 { 1403 .name = "pci_epf_vntb", 1404 }, 1405 {}, 1406 }; 1407 1408 static struct pci_epf_driver epf_ntb_driver = { 1409 .driver.name = "pci_epf_vntb", 1410 .probe = epf_ntb_probe, 1411 .id_table = epf_ntb_ids, 1412 .ops = &epf_ntb_ops, 1413 .owner = THIS_MODULE, 1414 }; 1415 1416 static int __init epf_ntb_init(void) 1417 { 1418 int ret; 1419 1420 kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM | 1421 WQ_HIGHPRI, 0); 1422 ret = pci_epf_register_driver(&epf_ntb_driver); 1423 if (ret) { 1424 destroy_workqueue(kpcintb_workqueue); 1425 pr_err("Failed to register pci epf ntb driver --> %d\n", ret); 1426 return ret; 1427 } 1428 1429 return 0; 1430 } 1431 module_init(epf_ntb_init); 1432 1433 static void __exit epf_ntb_exit(void) 1434 { 1435 pci_epf_unregister_driver(&epf_ntb_driver); 1436 destroy_workqueue(kpcintb_workqueue); 1437 } 1438 module_exit(epf_ntb_exit); 1439 1440 MODULE_DESCRIPTION("PCI EPF NTB DRIVER"); 1441 MODULE_AUTHOR("Frank Li <Frank.li@nxp.com>"); 1442 MODULE_LICENSE("GPL v2"); 1443