1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Test driver to test endpoint functionality 4 * 5 * Copyright (C) 2017 Texas Instruments 6 * Author: Kishon Vijay Abraham I <kishon@ti.com> 7 */ 8 9 #include <linux/crc32.h> 10 #include <linux/delay.h> 11 #include <linux/dmaengine.h> 12 #include <linux/io.h> 13 #include <linux/module.h> 14 #include <linux/slab.h> 15 #include <linux/pci_ids.h> 16 #include <linux/random.h> 17 18 #include <linux/pci-epc.h> 19 #include <linux/pci-epf.h> 20 #include <linux/pci_regs.h> 21 22 #define IRQ_TYPE_LEGACY 0 23 #define IRQ_TYPE_MSI 1 24 #define IRQ_TYPE_MSIX 2 25 26 #define COMMAND_RAISE_LEGACY_IRQ BIT(0) 27 #define COMMAND_RAISE_MSI_IRQ BIT(1) 28 #define COMMAND_RAISE_MSIX_IRQ BIT(2) 29 #define COMMAND_READ BIT(3) 30 #define COMMAND_WRITE BIT(4) 31 #define COMMAND_COPY BIT(5) 32 33 #define STATUS_READ_SUCCESS BIT(0) 34 #define STATUS_READ_FAIL BIT(1) 35 #define STATUS_WRITE_SUCCESS BIT(2) 36 #define STATUS_WRITE_FAIL BIT(3) 37 #define STATUS_COPY_SUCCESS BIT(4) 38 #define STATUS_COPY_FAIL BIT(5) 39 #define STATUS_IRQ_RAISED BIT(6) 40 #define STATUS_SRC_ADDR_INVALID BIT(7) 41 #define STATUS_DST_ADDR_INVALID BIT(8) 42 43 #define FLAG_USE_DMA BIT(0) 44 45 #define TIMER_RESOLUTION 1 46 47 static struct workqueue_struct *kpcitest_workqueue; 48 49 struct pci_epf_test { 50 void *reg[PCI_STD_NUM_BARS]; 51 struct pci_epf *epf; 52 enum pci_barno test_reg_bar; 53 size_t msix_table_offset; 54 struct delayed_work cmd_handler; 55 struct dma_chan *dma_chan; 56 struct completion transfer_complete; 57 bool dma_supported; 58 const struct pci_epc_features *epc_features; 59 }; 60 61 struct pci_epf_test_reg { 62 u32 magic; 63 u32 command; 64 u32 status; 65 u64 src_addr; 66 u64 dst_addr; 67 u32 size; 68 u32 checksum; 69 u32 irq_type; 70 u32 irq_number; 71 u32 flags; 72 } __packed; 73 74 static struct pci_epf_header test_header = { 75 .vendorid = PCI_ANY_ID, 76 .deviceid = PCI_ANY_ID, 77 .baseclass_code = PCI_CLASS_OTHERS, 78 .interrupt_pin = PCI_INTERRUPT_INTA, 79 }; 80 81 static size_t bar_size[] = { 512, 512, 1024, 16384, 131072, 1048576 }; 82 83 static void pci_epf_test_dma_callback(void *param) 84 { 85 struct pci_epf_test *epf_test = param; 86 87 complete(&epf_test->transfer_complete); 88 } 89 90 /** 91 * pci_epf_test_data_transfer() - Function that uses dmaengine API to transfer 92 * data between PCIe EP and remote PCIe RC 93 * @epf_test: the EPF test device that performs the data transfer operation 94 * @dma_dst: The destination address of the data transfer. It can be a physical 95 * address given by pci_epc_mem_alloc_addr or DMA mapping APIs. 96 * @dma_src: The source address of the data transfer. It can be a physical 97 * address given by pci_epc_mem_alloc_addr or DMA mapping APIs. 98 * @len: The size of the data transfer 99 * 100 * Function that uses dmaengine API to transfer data between PCIe EP and remote 101 * PCIe RC. The source and destination address can be a physical address given 102 * by pci_epc_mem_alloc_addr or the one obtained using DMA mapping APIs. 103 * 104 * The function returns '0' on success and negative value on failure. 105 */ 106 static int pci_epf_test_data_transfer(struct pci_epf_test *epf_test, 107 dma_addr_t dma_dst, dma_addr_t dma_src, 108 size_t len) 109 { 110 enum dma_ctrl_flags flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT; 111 struct dma_chan *chan = epf_test->dma_chan; 112 struct pci_epf *epf = epf_test->epf; 113 struct dma_async_tx_descriptor *tx; 114 struct device *dev = &epf->dev; 115 dma_cookie_t cookie; 116 int ret; 117 118 if (IS_ERR_OR_NULL(chan)) { 119 dev_err(dev, "Invalid DMA memcpy channel\n"); 120 return -EINVAL; 121 } 122 123 tx = dmaengine_prep_dma_memcpy(chan, dma_dst, dma_src, len, flags); 124 if (!tx) { 125 dev_err(dev, "Failed to prepare DMA memcpy\n"); 126 return -EIO; 127 } 128 129 tx->callback = pci_epf_test_dma_callback; 130 tx->callback_param = epf_test; 131 cookie = tx->tx_submit(tx); 132 reinit_completion(&epf_test->transfer_complete); 133 134 ret = dma_submit_error(cookie); 135 if (ret) { 136 dev_err(dev, "Failed to do DMA tx_submit %d\n", cookie); 137 return -EIO; 138 } 139 140 dma_async_issue_pending(chan); 141 ret = wait_for_completion_interruptible(&epf_test->transfer_complete); 142 if (ret < 0) { 143 dmaengine_terminate_sync(chan); 144 dev_err(dev, "DMA wait_for_completion_timeout\n"); 145 return -ETIMEDOUT; 146 } 147 148 return 0; 149 } 150 151 /** 152 * pci_epf_test_init_dma_chan() - Function to initialize EPF test DMA channel 153 * @epf_test: the EPF test device that performs data transfer operation 154 * 155 * Function to initialize EPF test DMA channel. 156 */ 157 static int pci_epf_test_init_dma_chan(struct pci_epf_test *epf_test) 158 { 159 struct pci_epf *epf = epf_test->epf; 160 struct device *dev = &epf->dev; 161 struct dma_chan *dma_chan; 162 dma_cap_mask_t mask; 163 int ret; 164 165 dma_cap_zero(mask); 166 dma_cap_set(DMA_MEMCPY, mask); 167 168 dma_chan = dma_request_chan_by_mask(&mask); 169 if (IS_ERR(dma_chan)) { 170 ret = PTR_ERR(dma_chan); 171 if (ret != -EPROBE_DEFER) 172 dev_err(dev, "Failed to get DMA channel\n"); 173 return ret; 174 } 175 init_completion(&epf_test->transfer_complete); 176 177 epf_test->dma_chan = dma_chan; 178 179 return 0; 180 } 181 182 /** 183 * pci_epf_test_clean_dma_chan() - Function to cleanup EPF test DMA channel 184 * @epf_test: the EPF test device that performs data transfer operation 185 * 186 * Helper to cleanup EPF test DMA channel. 187 */ 188 static void pci_epf_test_clean_dma_chan(struct pci_epf_test *epf_test) 189 { 190 if (!epf_test->dma_supported) 191 return; 192 193 dma_release_channel(epf_test->dma_chan); 194 epf_test->dma_chan = NULL; 195 } 196 197 static void pci_epf_test_print_rate(const char *ops, u64 size, 198 struct timespec64 *start, 199 struct timespec64 *end, bool dma) 200 { 201 struct timespec64 ts; 202 u64 rate, ns; 203 204 ts = timespec64_sub(*end, *start); 205 206 /* convert both size (stored in 'rate') and time in terms of 'ns' */ 207 ns = timespec64_to_ns(&ts); 208 rate = size * NSEC_PER_SEC; 209 210 /* Divide both size (stored in 'rate') and ns by a common factor */ 211 while (ns > UINT_MAX) { 212 rate >>= 1; 213 ns >>= 1; 214 } 215 216 if (!ns) 217 return; 218 219 /* calculate the rate */ 220 do_div(rate, (uint32_t)ns); 221 222 pr_info("\n%s => Size: %llu bytes\t DMA: %s\t Time: %llu.%09u seconds\t" 223 "Rate: %llu KB/s\n", ops, size, dma ? "YES" : "NO", 224 (u64)ts.tv_sec, (u32)ts.tv_nsec, rate / 1024); 225 } 226 227 static int pci_epf_test_copy(struct pci_epf_test *epf_test) 228 { 229 int ret; 230 bool use_dma; 231 void __iomem *src_addr; 232 void __iomem *dst_addr; 233 phys_addr_t src_phys_addr; 234 phys_addr_t dst_phys_addr; 235 struct timespec64 start, end; 236 struct pci_epf *epf = epf_test->epf; 237 struct device *dev = &epf->dev; 238 struct pci_epc *epc = epf->epc; 239 enum pci_barno test_reg_bar = epf_test->test_reg_bar; 240 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; 241 242 src_addr = pci_epc_mem_alloc_addr(epc, &src_phys_addr, reg->size); 243 if (!src_addr) { 244 dev_err(dev, "Failed to allocate source address\n"); 245 reg->status = STATUS_SRC_ADDR_INVALID; 246 ret = -ENOMEM; 247 goto err; 248 } 249 250 ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, src_phys_addr, 251 reg->src_addr, reg->size); 252 if (ret) { 253 dev_err(dev, "Failed to map source address\n"); 254 reg->status = STATUS_SRC_ADDR_INVALID; 255 goto err_src_addr; 256 } 257 258 dst_addr = pci_epc_mem_alloc_addr(epc, &dst_phys_addr, reg->size); 259 if (!dst_addr) { 260 dev_err(dev, "Failed to allocate destination address\n"); 261 reg->status = STATUS_DST_ADDR_INVALID; 262 ret = -ENOMEM; 263 goto err_src_map_addr; 264 } 265 266 ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, dst_phys_addr, 267 reg->dst_addr, reg->size); 268 if (ret) { 269 dev_err(dev, "Failed to map destination address\n"); 270 reg->status = STATUS_DST_ADDR_INVALID; 271 goto err_dst_addr; 272 } 273 274 ktime_get_ts64(&start); 275 use_dma = !!(reg->flags & FLAG_USE_DMA); 276 if (use_dma) { 277 if (!epf_test->dma_supported) { 278 dev_err(dev, "Cannot transfer data using DMA\n"); 279 ret = -EINVAL; 280 goto err_map_addr; 281 } 282 283 ret = pci_epf_test_data_transfer(epf_test, dst_phys_addr, 284 src_phys_addr, reg->size); 285 if (ret) 286 dev_err(dev, "Data transfer failed\n"); 287 } else { 288 void *buf; 289 290 buf = kzalloc(reg->size, GFP_KERNEL); 291 if (!buf) { 292 ret = -ENOMEM; 293 goto err_map_addr; 294 } 295 296 memcpy_fromio(buf, src_addr, reg->size); 297 memcpy_toio(dst_addr, buf, reg->size); 298 kfree(buf); 299 } 300 ktime_get_ts64(&end); 301 pci_epf_test_print_rate("COPY", reg->size, &start, &end, use_dma); 302 303 err_map_addr: 304 pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, dst_phys_addr); 305 306 err_dst_addr: 307 pci_epc_mem_free_addr(epc, dst_phys_addr, dst_addr, reg->size); 308 309 err_src_map_addr: 310 pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, src_phys_addr); 311 312 err_src_addr: 313 pci_epc_mem_free_addr(epc, src_phys_addr, src_addr, reg->size); 314 315 err: 316 return ret; 317 } 318 319 static int pci_epf_test_read(struct pci_epf_test *epf_test) 320 { 321 int ret; 322 void __iomem *src_addr; 323 void *buf; 324 u32 crc32; 325 bool use_dma; 326 phys_addr_t phys_addr; 327 phys_addr_t dst_phys_addr; 328 struct timespec64 start, end; 329 struct pci_epf *epf = epf_test->epf; 330 struct device *dev = &epf->dev; 331 struct pci_epc *epc = epf->epc; 332 struct device *dma_dev = epf->epc->dev.parent; 333 enum pci_barno test_reg_bar = epf_test->test_reg_bar; 334 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; 335 336 src_addr = pci_epc_mem_alloc_addr(epc, &phys_addr, reg->size); 337 if (!src_addr) { 338 dev_err(dev, "Failed to allocate address\n"); 339 reg->status = STATUS_SRC_ADDR_INVALID; 340 ret = -ENOMEM; 341 goto err; 342 } 343 344 ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, phys_addr, 345 reg->src_addr, reg->size); 346 if (ret) { 347 dev_err(dev, "Failed to map address\n"); 348 reg->status = STATUS_SRC_ADDR_INVALID; 349 goto err_addr; 350 } 351 352 buf = kzalloc(reg->size, GFP_KERNEL); 353 if (!buf) { 354 ret = -ENOMEM; 355 goto err_map_addr; 356 } 357 358 use_dma = !!(reg->flags & FLAG_USE_DMA); 359 if (use_dma) { 360 if (!epf_test->dma_supported) { 361 dev_err(dev, "Cannot transfer data using DMA\n"); 362 ret = -EINVAL; 363 goto err_dma_map; 364 } 365 366 dst_phys_addr = dma_map_single(dma_dev, buf, reg->size, 367 DMA_FROM_DEVICE); 368 if (dma_mapping_error(dma_dev, dst_phys_addr)) { 369 dev_err(dev, "Failed to map destination buffer addr\n"); 370 ret = -ENOMEM; 371 goto err_dma_map; 372 } 373 374 ktime_get_ts64(&start); 375 ret = pci_epf_test_data_transfer(epf_test, dst_phys_addr, 376 phys_addr, reg->size); 377 if (ret) 378 dev_err(dev, "Data transfer failed\n"); 379 ktime_get_ts64(&end); 380 381 dma_unmap_single(dma_dev, dst_phys_addr, reg->size, 382 DMA_FROM_DEVICE); 383 } else { 384 ktime_get_ts64(&start); 385 memcpy_fromio(buf, src_addr, reg->size); 386 ktime_get_ts64(&end); 387 } 388 389 pci_epf_test_print_rate("READ", reg->size, &start, &end, use_dma); 390 391 crc32 = crc32_le(~0, buf, reg->size); 392 if (crc32 != reg->checksum) 393 ret = -EIO; 394 395 err_dma_map: 396 kfree(buf); 397 398 err_map_addr: 399 pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, phys_addr); 400 401 err_addr: 402 pci_epc_mem_free_addr(epc, phys_addr, src_addr, reg->size); 403 404 err: 405 return ret; 406 } 407 408 static int pci_epf_test_write(struct pci_epf_test *epf_test) 409 { 410 int ret; 411 void __iomem *dst_addr; 412 void *buf; 413 bool use_dma; 414 phys_addr_t phys_addr; 415 phys_addr_t src_phys_addr; 416 struct timespec64 start, end; 417 struct pci_epf *epf = epf_test->epf; 418 struct device *dev = &epf->dev; 419 struct pci_epc *epc = epf->epc; 420 struct device *dma_dev = epf->epc->dev.parent; 421 enum pci_barno test_reg_bar = epf_test->test_reg_bar; 422 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; 423 424 dst_addr = pci_epc_mem_alloc_addr(epc, &phys_addr, reg->size); 425 if (!dst_addr) { 426 dev_err(dev, "Failed to allocate address\n"); 427 reg->status = STATUS_DST_ADDR_INVALID; 428 ret = -ENOMEM; 429 goto err; 430 } 431 432 ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, phys_addr, 433 reg->dst_addr, reg->size); 434 if (ret) { 435 dev_err(dev, "Failed to map address\n"); 436 reg->status = STATUS_DST_ADDR_INVALID; 437 goto err_addr; 438 } 439 440 buf = kzalloc(reg->size, GFP_KERNEL); 441 if (!buf) { 442 ret = -ENOMEM; 443 goto err_map_addr; 444 } 445 446 get_random_bytes(buf, reg->size); 447 reg->checksum = crc32_le(~0, buf, reg->size); 448 449 use_dma = !!(reg->flags & FLAG_USE_DMA); 450 if (use_dma) { 451 if (!epf_test->dma_supported) { 452 dev_err(dev, "Cannot transfer data using DMA\n"); 453 ret = -EINVAL; 454 goto err_dma_map; 455 } 456 457 src_phys_addr = dma_map_single(dma_dev, buf, reg->size, 458 DMA_TO_DEVICE); 459 if (dma_mapping_error(dma_dev, src_phys_addr)) { 460 dev_err(dev, "Failed to map source buffer addr\n"); 461 ret = -ENOMEM; 462 goto err_dma_map; 463 } 464 465 ktime_get_ts64(&start); 466 ret = pci_epf_test_data_transfer(epf_test, phys_addr, 467 src_phys_addr, reg->size); 468 if (ret) 469 dev_err(dev, "Data transfer failed\n"); 470 ktime_get_ts64(&end); 471 472 dma_unmap_single(dma_dev, src_phys_addr, reg->size, 473 DMA_TO_DEVICE); 474 } else { 475 ktime_get_ts64(&start); 476 memcpy_toio(dst_addr, buf, reg->size); 477 ktime_get_ts64(&end); 478 } 479 480 pci_epf_test_print_rate("WRITE", reg->size, &start, &end, use_dma); 481 482 /* 483 * wait 1ms inorder for the write to complete. Without this delay L3 484 * error in observed in the host system. 485 */ 486 usleep_range(1000, 2000); 487 488 err_dma_map: 489 kfree(buf); 490 491 err_map_addr: 492 pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, phys_addr); 493 494 err_addr: 495 pci_epc_mem_free_addr(epc, phys_addr, dst_addr, reg->size); 496 497 err: 498 return ret; 499 } 500 501 static void pci_epf_test_raise_irq(struct pci_epf_test *epf_test, u8 irq_type, 502 u16 irq) 503 { 504 struct pci_epf *epf = epf_test->epf; 505 struct device *dev = &epf->dev; 506 struct pci_epc *epc = epf->epc; 507 enum pci_barno test_reg_bar = epf_test->test_reg_bar; 508 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; 509 510 reg->status |= STATUS_IRQ_RAISED; 511 512 switch (irq_type) { 513 case IRQ_TYPE_LEGACY: 514 pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, 515 PCI_EPC_IRQ_LEGACY, 0); 516 break; 517 case IRQ_TYPE_MSI: 518 pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, 519 PCI_EPC_IRQ_MSI, irq); 520 break; 521 case IRQ_TYPE_MSIX: 522 pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, 523 PCI_EPC_IRQ_MSIX, irq); 524 break; 525 default: 526 dev_err(dev, "Failed to raise IRQ, unknown type\n"); 527 break; 528 } 529 } 530 531 static void pci_epf_test_cmd_handler(struct work_struct *work) 532 { 533 int ret; 534 int count; 535 u32 command; 536 struct pci_epf_test *epf_test = container_of(work, struct pci_epf_test, 537 cmd_handler.work); 538 struct pci_epf *epf = epf_test->epf; 539 struct device *dev = &epf->dev; 540 struct pci_epc *epc = epf->epc; 541 enum pci_barno test_reg_bar = epf_test->test_reg_bar; 542 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; 543 544 command = reg->command; 545 if (!command) 546 goto reset_handler; 547 548 reg->command = 0; 549 reg->status = 0; 550 551 if (reg->irq_type > IRQ_TYPE_MSIX) { 552 dev_err(dev, "Failed to detect IRQ type\n"); 553 goto reset_handler; 554 } 555 556 if (command & COMMAND_RAISE_LEGACY_IRQ) { 557 reg->status = STATUS_IRQ_RAISED; 558 pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, 559 PCI_EPC_IRQ_LEGACY, 0); 560 goto reset_handler; 561 } 562 563 if (command & COMMAND_WRITE) { 564 ret = pci_epf_test_write(epf_test); 565 if (ret) 566 reg->status |= STATUS_WRITE_FAIL; 567 else 568 reg->status |= STATUS_WRITE_SUCCESS; 569 pci_epf_test_raise_irq(epf_test, reg->irq_type, 570 reg->irq_number); 571 goto reset_handler; 572 } 573 574 if (command & COMMAND_READ) { 575 ret = pci_epf_test_read(epf_test); 576 if (!ret) 577 reg->status |= STATUS_READ_SUCCESS; 578 else 579 reg->status |= STATUS_READ_FAIL; 580 pci_epf_test_raise_irq(epf_test, reg->irq_type, 581 reg->irq_number); 582 goto reset_handler; 583 } 584 585 if (command & COMMAND_COPY) { 586 ret = pci_epf_test_copy(epf_test); 587 if (!ret) 588 reg->status |= STATUS_COPY_SUCCESS; 589 else 590 reg->status |= STATUS_COPY_FAIL; 591 pci_epf_test_raise_irq(epf_test, reg->irq_type, 592 reg->irq_number); 593 goto reset_handler; 594 } 595 596 if (command & COMMAND_RAISE_MSI_IRQ) { 597 count = pci_epc_get_msi(epc, epf->func_no, epf->vfunc_no); 598 if (reg->irq_number > count || count <= 0) 599 goto reset_handler; 600 reg->status = STATUS_IRQ_RAISED; 601 pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, 602 PCI_EPC_IRQ_MSI, reg->irq_number); 603 goto reset_handler; 604 } 605 606 if (command & COMMAND_RAISE_MSIX_IRQ) { 607 count = pci_epc_get_msix(epc, epf->func_no, epf->vfunc_no); 608 if (reg->irq_number > count || count <= 0) 609 goto reset_handler; 610 reg->status = STATUS_IRQ_RAISED; 611 pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, 612 PCI_EPC_IRQ_MSIX, reg->irq_number); 613 goto reset_handler; 614 } 615 616 reset_handler: 617 queue_delayed_work(kpcitest_workqueue, &epf_test->cmd_handler, 618 msecs_to_jiffies(1)); 619 } 620 621 static void pci_epf_test_unbind(struct pci_epf *epf) 622 { 623 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 624 struct pci_epc *epc = epf->epc; 625 struct pci_epf_bar *epf_bar; 626 int bar; 627 628 cancel_delayed_work(&epf_test->cmd_handler); 629 pci_epf_test_clean_dma_chan(epf_test); 630 pci_epc_stop(epc); 631 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) { 632 epf_bar = &epf->bar[bar]; 633 634 if (epf_test->reg[bar]) { 635 pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, 636 epf_bar); 637 pci_epf_free_space(epf, epf_test->reg[bar], bar, 638 PRIMARY_INTERFACE); 639 } 640 } 641 } 642 643 static int pci_epf_test_set_bar(struct pci_epf *epf) 644 { 645 int bar, add; 646 int ret; 647 struct pci_epf_bar *epf_bar; 648 struct pci_epc *epc = epf->epc; 649 struct device *dev = &epf->dev; 650 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 651 enum pci_barno test_reg_bar = epf_test->test_reg_bar; 652 const struct pci_epc_features *epc_features; 653 654 epc_features = epf_test->epc_features; 655 656 for (bar = 0; bar < PCI_STD_NUM_BARS; bar += add) { 657 epf_bar = &epf->bar[bar]; 658 /* 659 * pci_epc_set_bar() sets PCI_BASE_ADDRESS_MEM_TYPE_64 660 * if the specific implementation required a 64-bit BAR, 661 * even if we only requested a 32-bit BAR. 662 */ 663 add = (epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64) ? 2 : 1; 664 665 if (!!(epc_features->reserved_bar & (1 << bar))) 666 continue; 667 668 ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, 669 epf_bar); 670 if (ret) { 671 pci_epf_free_space(epf, epf_test->reg[bar], bar, 672 PRIMARY_INTERFACE); 673 dev_err(dev, "Failed to set BAR%d\n", bar); 674 if (bar == test_reg_bar) 675 return ret; 676 } 677 } 678 679 return 0; 680 } 681 682 static int pci_epf_test_core_init(struct pci_epf *epf) 683 { 684 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 685 struct pci_epf_header *header = epf->header; 686 const struct pci_epc_features *epc_features; 687 struct pci_epc *epc = epf->epc; 688 struct device *dev = &epf->dev; 689 bool msix_capable = false; 690 bool msi_capable = true; 691 int ret; 692 693 epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no); 694 if (epc_features) { 695 msix_capable = epc_features->msix_capable; 696 msi_capable = epc_features->msi_capable; 697 } 698 699 if (epf->vfunc_no <= 1) { 700 ret = pci_epc_write_header(epc, epf->func_no, epf->vfunc_no, header); 701 if (ret) { 702 dev_err(dev, "Configuration header write failed\n"); 703 return ret; 704 } 705 } 706 707 ret = pci_epf_test_set_bar(epf); 708 if (ret) 709 return ret; 710 711 if (msi_capable) { 712 ret = pci_epc_set_msi(epc, epf->func_no, epf->vfunc_no, 713 epf->msi_interrupts); 714 if (ret) { 715 dev_err(dev, "MSI configuration failed\n"); 716 return ret; 717 } 718 } 719 720 if (msix_capable) { 721 ret = pci_epc_set_msix(epc, epf->func_no, epf->vfunc_no, 722 epf->msix_interrupts, 723 epf_test->test_reg_bar, 724 epf_test->msix_table_offset); 725 if (ret) { 726 dev_err(dev, "MSI-X configuration failed\n"); 727 return ret; 728 } 729 } 730 731 return 0; 732 } 733 734 static int pci_epf_test_notifier(struct notifier_block *nb, unsigned long val, 735 void *data) 736 { 737 struct pci_epf *epf = container_of(nb, struct pci_epf, nb); 738 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 739 int ret; 740 741 switch (val) { 742 case CORE_INIT: 743 ret = pci_epf_test_core_init(epf); 744 if (ret) 745 return NOTIFY_BAD; 746 break; 747 748 case LINK_UP: 749 queue_delayed_work(kpcitest_workqueue, &epf_test->cmd_handler, 750 msecs_to_jiffies(1)); 751 break; 752 753 default: 754 dev_err(&epf->dev, "Invalid EPF test notifier event\n"); 755 return NOTIFY_BAD; 756 } 757 758 return NOTIFY_OK; 759 } 760 761 static int pci_epf_test_alloc_space(struct pci_epf *epf) 762 { 763 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 764 struct device *dev = &epf->dev; 765 struct pci_epf_bar *epf_bar; 766 size_t msix_table_size = 0; 767 size_t test_reg_bar_size; 768 size_t pba_size = 0; 769 bool msix_capable; 770 void *base; 771 int bar, add; 772 enum pci_barno test_reg_bar = epf_test->test_reg_bar; 773 const struct pci_epc_features *epc_features; 774 size_t test_reg_size; 775 776 epc_features = epf_test->epc_features; 777 778 test_reg_bar_size = ALIGN(sizeof(struct pci_epf_test_reg), 128); 779 780 msix_capable = epc_features->msix_capable; 781 if (msix_capable) { 782 msix_table_size = PCI_MSIX_ENTRY_SIZE * epf->msix_interrupts; 783 epf_test->msix_table_offset = test_reg_bar_size; 784 /* Align to QWORD or 8 Bytes */ 785 pba_size = ALIGN(DIV_ROUND_UP(epf->msix_interrupts, 8), 8); 786 } 787 test_reg_size = test_reg_bar_size + msix_table_size + pba_size; 788 789 if (epc_features->bar_fixed_size[test_reg_bar]) { 790 if (test_reg_size > bar_size[test_reg_bar]) 791 return -ENOMEM; 792 test_reg_size = bar_size[test_reg_bar]; 793 } 794 795 base = pci_epf_alloc_space(epf, test_reg_size, test_reg_bar, 796 epc_features->align, PRIMARY_INTERFACE); 797 if (!base) { 798 dev_err(dev, "Failed to allocated register space\n"); 799 return -ENOMEM; 800 } 801 epf_test->reg[test_reg_bar] = base; 802 803 for (bar = 0; bar < PCI_STD_NUM_BARS; bar += add) { 804 epf_bar = &epf->bar[bar]; 805 add = (epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64) ? 2 : 1; 806 807 if (bar == test_reg_bar) 808 continue; 809 810 if (!!(epc_features->reserved_bar & (1 << bar))) 811 continue; 812 813 base = pci_epf_alloc_space(epf, bar_size[bar], bar, 814 epc_features->align, 815 PRIMARY_INTERFACE); 816 if (!base) 817 dev_err(dev, "Failed to allocate space for BAR%d\n", 818 bar); 819 epf_test->reg[bar] = base; 820 } 821 822 return 0; 823 } 824 825 static void pci_epf_configure_bar(struct pci_epf *epf, 826 const struct pci_epc_features *epc_features) 827 { 828 struct pci_epf_bar *epf_bar; 829 bool bar_fixed_64bit; 830 int i; 831 832 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 833 epf_bar = &epf->bar[i]; 834 bar_fixed_64bit = !!(epc_features->bar_fixed_64bit & (1 << i)); 835 if (bar_fixed_64bit) 836 epf_bar->flags |= PCI_BASE_ADDRESS_MEM_TYPE_64; 837 if (epc_features->bar_fixed_size[i]) 838 bar_size[i] = epc_features->bar_fixed_size[i]; 839 } 840 } 841 842 static int pci_epf_test_bind(struct pci_epf *epf) 843 { 844 int ret; 845 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 846 const struct pci_epc_features *epc_features; 847 enum pci_barno test_reg_bar = BAR_0; 848 struct pci_epc *epc = epf->epc; 849 bool linkup_notifier = false; 850 bool core_init_notifier = false; 851 852 if (WARN_ON_ONCE(!epc)) 853 return -EINVAL; 854 855 epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no); 856 if (!epc_features) { 857 dev_err(&epf->dev, "epc_features not implemented\n"); 858 return -EOPNOTSUPP; 859 } 860 861 linkup_notifier = epc_features->linkup_notifier; 862 core_init_notifier = epc_features->core_init_notifier; 863 test_reg_bar = pci_epc_get_first_free_bar(epc_features); 864 if (test_reg_bar < 0) 865 return -EINVAL; 866 pci_epf_configure_bar(epf, epc_features); 867 868 epf_test->test_reg_bar = test_reg_bar; 869 epf_test->epc_features = epc_features; 870 871 ret = pci_epf_test_alloc_space(epf); 872 if (ret) 873 return ret; 874 875 if (!core_init_notifier) { 876 ret = pci_epf_test_core_init(epf); 877 if (ret) 878 return ret; 879 } 880 881 epf_test->dma_supported = true; 882 883 ret = pci_epf_test_init_dma_chan(epf_test); 884 if (ret) 885 epf_test->dma_supported = false; 886 887 if (linkup_notifier) { 888 epf->nb.notifier_call = pci_epf_test_notifier; 889 pci_epc_register_notifier(epc, &epf->nb); 890 } else { 891 queue_work(kpcitest_workqueue, &epf_test->cmd_handler.work); 892 } 893 894 return 0; 895 } 896 897 static const struct pci_epf_device_id pci_epf_test_ids[] = { 898 { 899 .name = "pci_epf_test", 900 }, 901 {}, 902 }; 903 904 static int pci_epf_test_probe(struct pci_epf *epf) 905 { 906 struct pci_epf_test *epf_test; 907 struct device *dev = &epf->dev; 908 909 epf_test = devm_kzalloc(dev, sizeof(*epf_test), GFP_KERNEL); 910 if (!epf_test) 911 return -ENOMEM; 912 913 epf->header = &test_header; 914 epf_test->epf = epf; 915 916 INIT_DELAYED_WORK(&epf_test->cmd_handler, pci_epf_test_cmd_handler); 917 918 epf_set_drvdata(epf, epf_test); 919 return 0; 920 } 921 922 static struct pci_epf_ops ops = { 923 .unbind = pci_epf_test_unbind, 924 .bind = pci_epf_test_bind, 925 }; 926 927 static struct pci_epf_driver test_driver = { 928 .driver.name = "pci_epf_test", 929 .probe = pci_epf_test_probe, 930 .id_table = pci_epf_test_ids, 931 .ops = &ops, 932 .owner = THIS_MODULE, 933 }; 934 935 static int __init pci_epf_test_init(void) 936 { 937 int ret; 938 939 kpcitest_workqueue = alloc_workqueue("kpcitest", 940 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 941 if (!kpcitest_workqueue) { 942 pr_err("Failed to allocate the kpcitest work queue\n"); 943 return -ENOMEM; 944 } 945 946 ret = pci_epf_register_driver(&test_driver); 947 if (ret) { 948 destroy_workqueue(kpcitest_workqueue); 949 pr_err("Failed to register pci epf test driver --> %d\n", ret); 950 return ret; 951 } 952 953 return 0; 954 } 955 module_init(pci_epf_test_init); 956 957 static void __exit pci_epf_test_exit(void) 958 { 959 if (kpcitest_workqueue) 960 destroy_workqueue(kpcitest_workqueue); 961 pci_epf_unregister_driver(&test_driver); 962 } 963 module_exit(pci_epf_test_exit); 964 965 MODULE_DESCRIPTION("PCI EPF TEST DRIVER"); 966 MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>"); 967 MODULE_LICENSE("GPL v2"); 968