1 // SPDX-License-Identifier: GPL-2.0 2 /** 3 * Test driver to test endpoint functionality 4 * 5 * Copyright (C) 2017 Texas Instruments 6 * Author: Kishon Vijay Abraham I <kishon@ti.com> 7 */ 8 9 #include <linux/crc32.h> 10 #include <linux/delay.h> 11 #include <linux/io.h> 12 #include <linux/module.h> 13 #include <linux/slab.h> 14 #include <linux/pci_ids.h> 15 #include <linux/random.h> 16 17 #include <linux/pci-epc.h> 18 #include <linux/pci-epf.h> 19 #include <linux/pci_regs.h> 20 21 #define IRQ_TYPE_LEGACY 0 22 #define IRQ_TYPE_MSI 1 23 #define IRQ_TYPE_MSIX 2 24 25 #define COMMAND_RAISE_LEGACY_IRQ BIT(0) 26 #define COMMAND_RAISE_MSI_IRQ BIT(1) 27 #define COMMAND_RAISE_MSIX_IRQ BIT(2) 28 #define COMMAND_READ BIT(3) 29 #define COMMAND_WRITE BIT(4) 30 #define COMMAND_COPY BIT(5) 31 32 #define STATUS_READ_SUCCESS BIT(0) 33 #define STATUS_READ_FAIL BIT(1) 34 #define STATUS_WRITE_SUCCESS BIT(2) 35 #define STATUS_WRITE_FAIL BIT(3) 36 #define STATUS_COPY_SUCCESS BIT(4) 37 #define STATUS_COPY_FAIL BIT(5) 38 #define STATUS_IRQ_RAISED BIT(6) 39 #define STATUS_SRC_ADDR_INVALID BIT(7) 40 #define STATUS_DST_ADDR_INVALID BIT(8) 41 42 #define TIMER_RESOLUTION 1 43 44 static struct workqueue_struct *kpcitest_workqueue; 45 46 struct pci_epf_test { 47 void *reg[6]; 48 struct pci_epf *epf; 49 enum pci_barno test_reg_bar; 50 bool linkup_notifier; 51 bool msix_available; 52 struct delayed_work cmd_handler; 53 }; 54 55 struct pci_epf_test_reg { 56 u32 magic; 57 u32 command; 58 u32 status; 59 u64 src_addr; 60 u64 dst_addr; 61 u32 size; 62 u32 checksum; 63 u32 irq_type; 64 u32 irq_number; 65 } __packed; 66 67 static struct pci_epf_header test_header = { 68 .vendorid = PCI_ANY_ID, 69 .deviceid = PCI_ANY_ID, 70 .baseclass_code = PCI_CLASS_OTHERS, 71 .interrupt_pin = PCI_INTERRUPT_INTA, 72 }; 73 74 struct pci_epf_test_data { 75 enum pci_barno test_reg_bar; 76 bool linkup_notifier; 77 }; 78 79 static size_t bar_size[] = { 512, 512, 1024, 16384, 131072, 1048576 }; 80 81 static int pci_epf_test_copy(struct pci_epf_test *epf_test) 82 { 83 int ret; 84 void __iomem *src_addr; 85 void __iomem *dst_addr; 86 phys_addr_t src_phys_addr; 87 phys_addr_t dst_phys_addr; 88 struct pci_epf *epf = epf_test->epf; 89 struct device *dev = &epf->dev; 90 struct pci_epc *epc = epf->epc; 91 enum pci_barno test_reg_bar = epf_test->test_reg_bar; 92 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; 93 94 src_addr = pci_epc_mem_alloc_addr(epc, &src_phys_addr, reg->size); 95 if (!src_addr) { 96 dev_err(dev, "Failed to allocate source address\n"); 97 reg->status = STATUS_SRC_ADDR_INVALID; 98 ret = -ENOMEM; 99 goto err; 100 } 101 102 ret = pci_epc_map_addr(epc, epf->func_no, src_phys_addr, reg->src_addr, 103 reg->size); 104 if (ret) { 105 dev_err(dev, "Failed to map source address\n"); 106 reg->status = STATUS_SRC_ADDR_INVALID; 107 goto err_src_addr; 108 } 109 110 dst_addr = pci_epc_mem_alloc_addr(epc, &dst_phys_addr, reg->size); 111 if (!dst_addr) { 112 dev_err(dev, "Failed to allocate destination address\n"); 113 reg->status = STATUS_DST_ADDR_INVALID; 114 ret = -ENOMEM; 115 goto err_src_map_addr; 116 } 117 118 ret = pci_epc_map_addr(epc, epf->func_no, dst_phys_addr, reg->dst_addr, 119 reg->size); 120 if (ret) { 121 dev_err(dev, "Failed to map destination address\n"); 122 reg->status = STATUS_DST_ADDR_INVALID; 123 goto err_dst_addr; 124 } 125 126 memcpy(dst_addr, src_addr, reg->size); 127 128 pci_epc_unmap_addr(epc, epf->func_no, dst_phys_addr); 129 130 err_dst_addr: 131 pci_epc_mem_free_addr(epc, dst_phys_addr, dst_addr, reg->size); 132 133 err_src_map_addr: 134 pci_epc_unmap_addr(epc, epf->func_no, src_phys_addr); 135 136 err_src_addr: 137 pci_epc_mem_free_addr(epc, src_phys_addr, src_addr, reg->size); 138 139 err: 140 return ret; 141 } 142 143 static int pci_epf_test_read(struct pci_epf_test *epf_test) 144 { 145 int ret; 146 void __iomem *src_addr; 147 void *buf; 148 u32 crc32; 149 phys_addr_t phys_addr; 150 struct pci_epf *epf = epf_test->epf; 151 struct device *dev = &epf->dev; 152 struct pci_epc *epc = epf->epc; 153 enum pci_barno test_reg_bar = epf_test->test_reg_bar; 154 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; 155 156 src_addr = pci_epc_mem_alloc_addr(epc, &phys_addr, reg->size); 157 if (!src_addr) { 158 dev_err(dev, "Failed to allocate address\n"); 159 reg->status = STATUS_SRC_ADDR_INVALID; 160 ret = -ENOMEM; 161 goto err; 162 } 163 164 ret = pci_epc_map_addr(epc, epf->func_no, phys_addr, reg->src_addr, 165 reg->size); 166 if (ret) { 167 dev_err(dev, "Failed to map address\n"); 168 reg->status = STATUS_SRC_ADDR_INVALID; 169 goto err_addr; 170 } 171 172 buf = kzalloc(reg->size, GFP_KERNEL); 173 if (!buf) { 174 ret = -ENOMEM; 175 goto err_map_addr; 176 } 177 178 memcpy(buf, src_addr, reg->size); 179 180 crc32 = crc32_le(~0, buf, reg->size); 181 if (crc32 != reg->checksum) 182 ret = -EIO; 183 184 kfree(buf); 185 186 err_map_addr: 187 pci_epc_unmap_addr(epc, epf->func_no, phys_addr); 188 189 err_addr: 190 pci_epc_mem_free_addr(epc, phys_addr, src_addr, reg->size); 191 192 err: 193 return ret; 194 } 195 196 static int pci_epf_test_write(struct pci_epf_test *epf_test) 197 { 198 int ret; 199 void __iomem *dst_addr; 200 void *buf; 201 phys_addr_t phys_addr; 202 struct pci_epf *epf = epf_test->epf; 203 struct device *dev = &epf->dev; 204 struct pci_epc *epc = epf->epc; 205 enum pci_barno test_reg_bar = epf_test->test_reg_bar; 206 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; 207 208 dst_addr = pci_epc_mem_alloc_addr(epc, &phys_addr, reg->size); 209 if (!dst_addr) { 210 dev_err(dev, "Failed to allocate address\n"); 211 reg->status = STATUS_DST_ADDR_INVALID; 212 ret = -ENOMEM; 213 goto err; 214 } 215 216 ret = pci_epc_map_addr(epc, epf->func_no, phys_addr, reg->dst_addr, 217 reg->size); 218 if (ret) { 219 dev_err(dev, "Failed to map address\n"); 220 reg->status = STATUS_DST_ADDR_INVALID; 221 goto err_addr; 222 } 223 224 buf = kzalloc(reg->size, GFP_KERNEL); 225 if (!buf) { 226 ret = -ENOMEM; 227 goto err_map_addr; 228 } 229 230 get_random_bytes(buf, reg->size); 231 reg->checksum = crc32_le(~0, buf, reg->size); 232 233 memcpy(dst_addr, buf, reg->size); 234 235 /* 236 * wait 1ms inorder for the write to complete. Without this delay L3 237 * error in observed in the host system. 238 */ 239 usleep_range(1000, 2000); 240 241 kfree(buf); 242 243 err_map_addr: 244 pci_epc_unmap_addr(epc, epf->func_no, phys_addr); 245 246 err_addr: 247 pci_epc_mem_free_addr(epc, phys_addr, dst_addr, reg->size); 248 249 err: 250 return ret; 251 } 252 253 static void pci_epf_test_raise_irq(struct pci_epf_test *epf_test, u8 irq_type, 254 u16 irq) 255 { 256 struct pci_epf *epf = epf_test->epf; 257 struct device *dev = &epf->dev; 258 struct pci_epc *epc = epf->epc; 259 enum pci_barno test_reg_bar = epf_test->test_reg_bar; 260 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; 261 262 reg->status |= STATUS_IRQ_RAISED; 263 264 switch (irq_type) { 265 case IRQ_TYPE_LEGACY: 266 pci_epc_raise_irq(epc, epf->func_no, PCI_EPC_IRQ_LEGACY, 0); 267 break; 268 case IRQ_TYPE_MSI: 269 pci_epc_raise_irq(epc, epf->func_no, PCI_EPC_IRQ_MSI, irq); 270 break; 271 case IRQ_TYPE_MSIX: 272 pci_epc_raise_irq(epc, epf->func_no, PCI_EPC_IRQ_MSIX, irq); 273 break; 274 default: 275 dev_err(dev, "Failed to raise IRQ, unknown type\n"); 276 break; 277 } 278 } 279 280 static void pci_epf_test_cmd_handler(struct work_struct *work) 281 { 282 int ret; 283 int count; 284 u32 command; 285 struct pci_epf_test *epf_test = container_of(work, struct pci_epf_test, 286 cmd_handler.work); 287 struct pci_epf *epf = epf_test->epf; 288 struct device *dev = &epf->dev; 289 struct pci_epc *epc = epf->epc; 290 enum pci_barno test_reg_bar = epf_test->test_reg_bar; 291 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; 292 293 command = reg->command; 294 if (!command) 295 goto reset_handler; 296 297 reg->command = 0; 298 reg->status = 0; 299 300 if (reg->irq_type > IRQ_TYPE_MSIX) { 301 dev_err(dev, "Failed to detect IRQ type\n"); 302 goto reset_handler; 303 } 304 305 if (command & COMMAND_RAISE_LEGACY_IRQ) { 306 reg->status = STATUS_IRQ_RAISED; 307 pci_epc_raise_irq(epc, epf->func_no, PCI_EPC_IRQ_LEGACY, 0); 308 goto reset_handler; 309 } 310 311 if (command & COMMAND_WRITE) { 312 ret = pci_epf_test_write(epf_test); 313 if (ret) 314 reg->status |= STATUS_WRITE_FAIL; 315 else 316 reg->status |= STATUS_WRITE_SUCCESS; 317 pci_epf_test_raise_irq(epf_test, reg->irq_type, 318 reg->irq_number); 319 goto reset_handler; 320 } 321 322 if (command & COMMAND_READ) { 323 ret = pci_epf_test_read(epf_test); 324 if (!ret) 325 reg->status |= STATUS_READ_SUCCESS; 326 else 327 reg->status |= STATUS_READ_FAIL; 328 pci_epf_test_raise_irq(epf_test, reg->irq_type, 329 reg->irq_number); 330 goto reset_handler; 331 } 332 333 if (command & COMMAND_COPY) { 334 ret = pci_epf_test_copy(epf_test); 335 if (!ret) 336 reg->status |= STATUS_COPY_SUCCESS; 337 else 338 reg->status |= STATUS_COPY_FAIL; 339 pci_epf_test_raise_irq(epf_test, reg->irq_type, 340 reg->irq_number); 341 goto reset_handler; 342 } 343 344 if (command & COMMAND_RAISE_MSI_IRQ) { 345 count = pci_epc_get_msi(epc, epf->func_no); 346 if (reg->irq_number > count || count <= 0) 347 goto reset_handler; 348 reg->status = STATUS_IRQ_RAISED; 349 pci_epc_raise_irq(epc, epf->func_no, PCI_EPC_IRQ_MSI, 350 reg->irq_number); 351 goto reset_handler; 352 } 353 354 if (command & COMMAND_RAISE_MSIX_IRQ) { 355 count = pci_epc_get_msix(epc, epf->func_no); 356 if (reg->irq_number > count || count <= 0) 357 goto reset_handler; 358 reg->status = STATUS_IRQ_RAISED; 359 pci_epc_raise_irq(epc, epf->func_no, PCI_EPC_IRQ_MSIX, 360 reg->irq_number); 361 goto reset_handler; 362 } 363 364 reset_handler: 365 queue_delayed_work(kpcitest_workqueue, &epf_test->cmd_handler, 366 msecs_to_jiffies(1)); 367 } 368 369 static void pci_epf_test_linkup(struct pci_epf *epf) 370 { 371 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 372 373 queue_delayed_work(kpcitest_workqueue, &epf_test->cmd_handler, 374 msecs_to_jiffies(1)); 375 } 376 377 static void pci_epf_test_unbind(struct pci_epf *epf) 378 { 379 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 380 struct pci_epc *epc = epf->epc; 381 struct pci_epf_bar *epf_bar; 382 int bar; 383 384 cancel_delayed_work(&epf_test->cmd_handler); 385 pci_epc_stop(epc); 386 for (bar = BAR_0; bar <= BAR_5; bar++) { 387 epf_bar = &epf->bar[bar]; 388 389 if (epf_test->reg[bar]) { 390 pci_epf_free_space(epf, epf_test->reg[bar], bar); 391 pci_epc_clear_bar(epc, epf->func_no, epf_bar); 392 } 393 } 394 } 395 396 static int pci_epf_test_set_bar(struct pci_epf *epf) 397 { 398 int bar; 399 int ret; 400 struct pci_epf_bar *epf_bar; 401 struct pci_epc *epc = epf->epc; 402 struct device *dev = &epf->dev; 403 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 404 enum pci_barno test_reg_bar = epf_test->test_reg_bar; 405 406 for (bar = BAR_0; bar <= BAR_5; bar++) { 407 epf_bar = &epf->bar[bar]; 408 409 epf_bar->flags |= upper_32_bits(epf_bar->size) ? 410 PCI_BASE_ADDRESS_MEM_TYPE_64 : 411 PCI_BASE_ADDRESS_MEM_TYPE_32; 412 413 ret = pci_epc_set_bar(epc, epf->func_no, epf_bar); 414 if (ret) { 415 pci_epf_free_space(epf, epf_test->reg[bar], bar); 416 dev_err(dev, "Failed to set BAR%d\n", bar); 417 if (bar == test_reg_bar) 418 return ret; 419 } 420 /* 421 * pci_epc_set_bar() sets PCI_BASE_ADDRESS_MEM_TYPE_64 422 * if the specific implementation required a 64-bit BAR, 423 * even if we only requested a 32-bit BAR. 424 */ 425 if (epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64) 426 bar++; 427 } 428 429 return 0; 430 } 431 432 static int pci_epf_test_alloc_space(struct pci_epf *epf) 433 { 434 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 435 struct device *dev = &epf->dev; 436 void *base; 437 int bar; 438 enum pci_barno test_reg_bar = epf_test->test_reg_bar; 439 440 base = pci_epf_alloc_space(epf, sizeof(struct pci_epf_test_reg), 441 test_reg_bar); 442 if (!base) { 443 dev_err(dev, "Failed to allocated register space\n"); 444 return -ENOMEM; 445 } 446 epf_test->reg[test_reg_bar] = base; 447 448 for (bar = BAR_0; bar <= BAR_5; bar++) { 449 if (bar == test_reg_bar) 450 continue; 451 base = pci_epf_alloc_space(epf, bar_size[bar], bar); 452 if (!base) 453 dev_err(dev, "Failed to allocate space for BAR%d\n", 454 bar); 455 epf_test->reg[bar] = base; 456 } 457 458 return 0; 459 } 460 461 static int pci_epf_test_bind(struct pci_epf *epf) 462 { 463 int ret; 464 struct pci_epf_test *epf_test = epf_get_drvdata(epf); 465 struct pci_epf_header *header = epf->header; 466 struct pci_epc *epc = epf->epc; 467 struct device *dev = &epf->dev; 468 469 if (WARN_ON_ONCE(!epc)) 470 return -EINVAL; 471 472 if (epc->features & EPC_FEATURE_NO_LINKUP_NOTIFIER) 473 epf_test->linkup_notifier = false; 474 else 475 epf_test->linkup_notifier = true; 476 477 epf_test->msix_available = epc->features & EPC_FEATURE_MSIX_AVAILABLE; 478 479 epf_test->test_reg_bar = EPC_FEATURE_GET_BAR(epc->features); 480 481 ret = pci_epc_write_header(epc, epf->func_no, header); 482 if (ret) { 483 dev_err(dev, "Configuration header write failed\n"); 484 return ret; 485 } 486 487 ret = pci_epf_test_alloc_space(epf); 488 if (ret) 489 return ret; 490 491 ret = pci_epf_test_set_bar(epf); 492 if (ret) 493 return ret; 494 495 ret = pci_epc_set_msi(epc, epf->func_no, epf->msi_interrupts); 496 if (ret) { 497 dev_err(dev, "MSI configuration failed\n"); 498 return ret; 499 } 500 501 if (epf_test->msix_available) { 502 ret = pci_epc_set_msix(epc, epf->func_no, epf->msix_interrupts); 503 if (ret) { 504 dev_err(dev, "MSI-X configuration failed\n"); 505 return ret; 506 } 507 } 508 509 if (!epf_test->linkup_notifier) 510 queue_work(kpcitest_workqueue, &epf_test->cmd_handler.work); 511 512 return 0; 513 } 514 515 static const struct pci_epf_device_id pci_epf_test_ids[] = { 516 { 517 .name = "pci_epf_test", 518 }, 519 {}, 520 }; 521 522 static int pci_epf_test_probe(struct pci_epf *epf) 523 { 524 struct pci_epf_test *epf_test; 525 struct device *dev = &epf->dev; 526 const struct pci_epf_device_id *match; 527 struct pci_epf_test_data *data; 528 enum pci_barno test_reg_bar = BAR_0; 529 bool linkup_notifier = true; 530 531 match = pci_epf_match_device(pci_epf_test_ids, epf); 532 data = (struct pci_epf_test_data *)match->driver_data; 533 if (data) { 534 test_reg_bar = data->test_reg_bar; 535 linkup_notifier = data->linkup_notifier; 536 } 537 538 epf_test = devm_kzalloc(dev, sizeof(*epf_test), GFP_KERNEL); 539 if (!epf_test) 540 return -ENOMEM; 541 542 epf->header = &test_header; 543 epf_test->epf = epf; 544 epf_test->test_reg_bar = test_reg_bar; 545 epf_test->linkup_notifier = linkup_notifier; 546 547 INIT_DELAYED_WORK(&epf_test->cmd_handler, pci_epf_test_cmd_handler); 548 549 epf_set_drvdata(epf, epf_test); 550 return 0; 551 } 552 553 static struct pci_epf_ops ops = { 554 .unbind = pci_epf_test_unbind, 555 .bind = pci_epf_test_bind, 556 .linkup = pci_epf_test_linkup, 557 }; 558 559 static struct pci_epf_driver test_driver = { 560 .driver.name = "pci_epf_test", 561 .probe = pci_epf_test_probe, 562 .id_table = pci_epf_test_ids, 563 .ops = &ops, 564 .owner = THIS_MODULE, 565 }; 566 567 static int __init pci_epf_test_init(void) 568 { 569 int ret; 570 571 kpcitest_workqueue = alloc_workqueue("kpcitest", 572 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 573 ret = pci_epf_register_driver(&test_driver); 574 if (ret) { 575 pr_err("Failed to register pci epf test driver --> %d\n", ret); 576 return ret; 577 } 578 579 return 0; 580 } 581 module_init(pci_epf_test_init); 582 583 static void __exit pci_epf_test_exit(void) 584 { 585 pci_epf_unregister_driver(&test_driver); 586 } 587 module_exit(pci_epf_test_exit); 588 589 MODULE_DESCRIPTION("PCI EPF TEST DRIVER"); 590 MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>"); 591 MODULE_LICENSE("GPL v2"); 592