1 /* 2 * This file is provided under a dual BSD/GPLv2 license. When using or 3 * redistributing this file, you may do so under either license. 4 * 5 * GPL LICENSE SUMMARY 6 * 7 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved. 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * BSD LICENSE 14 * 15 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 21 * * Redistributions of source code must retain the above copyright 22 * notice, this list of conditions and the following disclaimer. 23 * * Redistributions in binary form must reproduce the above copy 24 * notice, this list of conditions and the following disclaimer in 25 * the documentation and/or other materials provided with the 26 * distribution. 27 * * Neither the name of AMD Corporation nor the names of its 28 * contributors may be used to endorse or promote products derived 29 * from this software without specific prior written permission. 30 * 31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 32 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 33 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 34 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 35 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 36 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 37 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 42 * 43 * AMD PCIe NTB Linux driver 44 * 45 * Contact Information: 46 * Xiangliang Yu <Xiangliang.Yu@amd.com> 47 */ 48 49 #include <linux/debugfs.h> 50 #include <linux/delay.h> 51 #include <linux/init.h> 52 #include <linux/interrupt.h> 53 #include <linux/module.h> 54 #include <linux/acpi.h> 55 #include <linux/pci.h> 56 #include <linux/random.h> 57 #include <linux/slab.h> 58 #include <linux/ntb.h> 59 60 #include "ntb_hw_amd.h" 61 62 #define NTB_NAME "ntb_hw_amd" 63 #define NTB_DESC "AMD(R) PCI-E Non-Transparent Bridge Driver" 64 #define NTB_VER "1.0" 65 66 MODULE_DESCRIPTION(NTB_DESC); 67 MODULE_VERSION(NTB_VER); 68 MODULE_LICENSE("Dual BSD/GPL"); 69 MODULE_AUTHOR("AMD Inc."); 70 71 static const struct file_operations amd_ntb_debugfs_info; 72 static struct dentry *debugfs_dir; 73 74 static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx) 75 { 76 if (idx < 0 || idx > ndev->mw_count) 77 return -EINVAL; 78 79 return 1 << idx; 80 } 81 82 static int amd_ntb_mw_count(struct ntb_dev *ntb) 83 { 84 return ntb_ndev(ntb)->mw_count; 85 } 86 87 static int amd_ntb_mw_get_range(struct ntb_dev *ntb, int idx, 88 phys_addr_t *base, 89 resource_size_t *size, 90 resource_size_t *align, 91 resource_size_t *align_size) 92 { 93 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 94 int bar; 95 96 bar = ndev_mw_to_bar(ndev, idx); 97 if (bar < 0) 98 return bar; 99 100 if (base) 101 *base = pci_resource_start(ndev->ntb.pdev, bar); 102 103 if (size) 104 *size = pci_resource_len(ndev->ntb.pdev, bar); 105 106 if (align) 107 *align = SZ_4K; 108 109 if (align_size) 110 *align_size = 1; 111 112 return 0; 113 } 114 115 static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int idx, 116 dma_addr_t addr, resource_size_t size) 117 { 118 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 119 unsigned long xlat_reg, limit_reg = 0; 120 resource_size_t mw_size; 121 void __iomem *mmio, *peer_mmio; 122 u64 base_addr, limit, reg_val; 123 int bar; 124 125 bar = ndev_mw_to_bar(ndev, idx); 126 if (bar < 0) 127 return bar; 128 129 mw_size = pci_resource_len(ndev->ntb.pdev, bar); 130 131 /* make sure the range fits in the usable mw size */ 132 if (size > mw_size) 133 return -EINVAL; 134 135 mmio = ndev->self_mmio; 136 peer_mmio = ndev->peer_mmio; 137 138 base_addr = pci_resource_start(ndev->ntb.pdev, bar); 139 140 if (bar != 1) { 141 xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 2); 142 limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 2); 143 144 /* Set the limit if supported */ 145 limit = size; 146 147 /* set and verify setting the translation address */ 148 write64(addr, peer_mmio + xlat_reg); 149 reg_val = read64(peer_mmio + xlat_reg); 150 if (reg_val != addr) { 151 write64(0, peer_mmio + xlat_reg); 152 return -EIO; 153 } 154 155 /* set and verify setting the limit */ 156 write64(limit, mmio + limit_reg); 157 reg_val = read64(mmio + limit_reg); 158 if (reg_val != limit) { 159 write64(base_addr, mmio + limit_reg); 160 write64(0, peer_mmio + xlat_reg); 161 return -EIO; 162 } 163 } else { 164 xlat_reg = AMD_BAR1XLAT_OFFSET; 165 limit_reg = AMD_BAR1LMT_OFFSET; 166 167 /* Set the limit if supported */ 168 limit = size; 169 170 /* set and verify setting the translation address */ 171 write64(addr, peer_mmio + xlat_reg); 172 reg_val = read64(peer_mmio + xlat_reg); 173 if (reg_val != addr) { 174 write64(0, peer_mmio + xlat_reg); 175 return -EIO; 176 } 177 178 /* set and verify setting the limit */ 179 writel(limit, mmio + limit_reg); 180 reg_val = readl(mmio + limit_reg); 181 if (reg_val != limit) { 182 writel(base_addr, mmio + limit_reg); 183 writel(0, peer_mmio + xlat_reg); 184 return -EIO; 185 } 186 } 187 188 return 0; 189 } 190 191 static int amd_link_is_up(struct amd_ntb_dev *ndev) 192 { 193 if (!ndev->peer_sta) 194 return NTB_LNK_STA_ACTIVE(ndev->cntl_sta); 195 196 if (ndev->peer_sta & AMD_LINK_UP_EVENT) { 197 ndev->peer_sta = 0; 198 return 1; 199 } 200 201 /* If peer_sta is reset or D0 event, the ISR has 202 * started a timer to check link status of hardware. 203 * So here just clear status bit. And if peer_sta is 204 * D3 or PME_TO, D0/reset event will be happened when 205 * system wakeup/poweron, so do nothing here. 206 */ 207 if (ndev->peer_sta & AMD_PEER_RESET_EVENT) 208 ndev->peer_sta &= ~AMD_PEER_RESET_EVENT; 209 else if (ndev->peer_sta & (AMD_PEER_D0_EVENT | AMD_LINK_DOWN_EVENT)) 210 ndev->peer_sta = 0; 211 212 return 0; 213 } 214 215 static int amd_ntb_link_is_up(struct ntb_dev *ntb, 216 enum ntb_speed *speed, 217 enum ntb_width *width) 218 { 219 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 220 int ret = 0; 221 222 if (amd_link_is_up(ndev)) { 223 if (speed) 224 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta); 225 if (width) 226 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta); 227 228 dev_dbg(ndev_dev(ndev), "link is up.\n"); 229 230 ret = 1; 231 } else { 232 if (speed) 233 *speed = NTB_SPEED_NONE; 234 if (width) 235 *width = NTB_WIDTH_NONE; 236 237 dev_dbg(ndev_dev(ndev), "link is down.\n"); 238 } 239 240 return ret; 241 } 242 243 static int amd_ntb_link_enable(struct ntb_dev *ntb, 244 enum ntb_speed max_speed, 245 enum ntb_width max_width) 246 { 247 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 248 void __iomem *mmio = ndev->self_mmio; 249 u32 ntb_ctl; 250 251 /* Enable event interrupt */ 252 ndev->int_mask &= ~AMD_EVENT_INTMASK; 253 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 254 255 if (ndev->ntb.topo == NTB_TOPO_SEC) 256 return -EINVAL; 257 dev_dbg(ndev_dev(ndev), "Enabling Link.\n"); 258 259 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET); 260 ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL); 261 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET); 262 263 return 0; 264 } 265 266 static int amd_ntb_link_disable(struct ntb_dev *ntb) 267 { 268 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 269 void __iomem *mmio = ndev->self_mmio; 270 u32 ntb_ctl; 271 272 /* Disable event interrupt */ 273 ndev->int_mask |= AMD_EVENT_INTMASK; 274 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 275 276 if (ndev->ntb.topo == NTB_TOPO_SEC) 277 return -EINVAL; 278 dev_dbg(ndev_dev(ndev), "Enabling Link.\n"); 279 280 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET); 281 ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL); 282 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET); 283 284 return 0; 285 } 286 287 static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb) 288 { 289 return ntb_ndev(ntb)->db_valid_mask; 290 } 291 292 static int amd_ntb_db_vector_count(struct ntb_dev *ntb) 293 { 294 return ntb_ndev(ntb)->db_count; 295 } 296 297 static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) 298 { 299 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 300 301 if (db_vector < 0 || db_vector > ndev->db_count) 302 return 0; 303 304 return ntb_ndev(ntb)->db_valid_mask & (1 << db_vector); 305 } 306 307 static u64 amd_ntb_db_read(struct ntb_dev *ntb) 308 { 309 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 310 void __iomem *mmio = ndev->self_mmio; 311 312 return (u64)readw(mmio + AMD_DBSTAT_OFFSET); 313 } 314 315 static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) 316 { 317 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 318 void __iomem *mmio = ndev->self_mmio; 319 320 writew((u16)db_bits, mmio + AMD_DBSTAT_OFFSET); 321 322 return 0; 323 } 324 325 static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) 326 { 327 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 328 void __iomem *mmio = ndev->self_mmio; 329 unsigned long flags; 330 331 if (db_bits & ~ndev->db_valid_mask) 332 return -EINVAL; 333 334 spin_lock_irqsave(&ndev->db_mask_lock, flags); 335 ndev->db_mask |= db_bits; 336 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET); 337 spin_unlock_irqrestore(&ndev->db_mask_lock, flags); 338 339 return 0; 340 } 341 342 static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) 343 { 344 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 345 void __iomem *mmio = ndev->self_mmio; 346 unsigned long flags; 347 348 if (db_bits & ~ndev->db_valid_mask) 349 return -EINVAL; 350 351 spin_lock_irqsave(&ndev->db_mask_lock, flags); 352 ndev->db_mask &= ~db_bits; 353 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET); 354 spin_unlock_irqrestore(&ndev->db_mask_lock, flags); 355 356 return 0; 357 } 358 359 static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) 360 { 361 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 362 void __iomem *mmio = ndev->self_mmio; 363 364 writew((u16)db_bits, mmio + AMD_DBREQ_OFFSET); 365 366 return 0; 367 } 368 369 static int amd_ntb_spad_count(struct ntb_dev *ntb) 370 { 371 return ntb_ndev(ntb)->spad_count; 372 } 373 374 static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx) 375 { 376 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 377 void __iomem *mmio = ndev->self_mmio; 378 u32 offset; 379 380 if (idx < 0 || idx >= ndev->spad_count) 381 return 0; 382 383 offset = ndev->self_spad + (idx << 2); 384 return readl(mmio + AMD_SPAD_OFFSET + offset); 385 } 386 387 static int amd_ntb_spad_write(struct ntb_dev *ntb, 388 int idx, u32 val) 389 { 390 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 391 void __iomem *mmio = ndev->self_mmio; 392 u32 offset; 393 394 if (idx < 0 || idx >= ndev->spad_count) 395 return -EINVAL; 396 397 offset = ndev->self_spad + (idx << 2); 398 writel(val, mmio + AMD_SPAD_OFFSET + offset); 399 400 return 0; 401 } 402 403 static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int idx) 404 { 405 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 406 void __iomem *mmio = ndev->self_mmio; 407 u32 offset; 408 409 if (idx < 0 || idx >= ndev->spad_count) 410 return -EINVAL; 411 412 offset = ndev->peer_spad + (idx << 2); 413 return readl(mmio + AMD_SPAD_OFFSET + offset); 414 } 415 416 static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, 417 int idx, u32 val) 418 { 419 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 420 void __iomem *mmio = ndev->self_mmio; 421 u32 offset; 422 423 if (idx < 0 || idx >= ndev->spad_count) 424 return -EINVAL; 425 426 offset = ndev->peer_spad + (idx << 2); 427 writel(val, mmio + AMD_SPAD_OFFSET + offset); 428 429 return 0; 430 } 431 432 static const struct ntb_dev_ops amd_ntb_ops = { 433 .mw_count = amd_ntb_mw_count, 434 .mw_get_range = amd_ntb_mw_get_range, 435 .mw_set_trans = amd_ntb_mw_set_trans, 436 .link_is_up = amd_ntb_link_is_up, 437 .link_enable = amd_ntb_link_enable, 438 .link_disable = amd_ntb_link_disable, 439 .db_valid_mask = amd_ntb_db_valid_mask, 440 .db_vector_count = amd_ntb_db_vector_count, 441 .db_vector_mask = amd_ntb_db_vector_mask, 442 .db_read = amd_ntb_db_read, 443 .db_clear = amd_ntb_db_clear, 444 .db_set_mask = amd_ntb_db_set_mask, 445 .db_clear_mask = amd_ntb_db_clear_mask, 446 .peer_db_set = amd_ntb_peer_db_set, 447 .spad_count = amd_ntb_spad_count, 448 .spad_read = amd_ntb_spad_read, 449 .spad_write = amd_ntb_spad_write, 450 .peer_spad_read = amd_ntb_peer_spad_read, 451 .peer_spad_write = amd_ntb_peer_spad_write, 452 }; 453 454 static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit) 455 { 456 void __iomem *mmio = ndev->self_mmio; 457 int reg; 458 459 reg = readl(mmio + AMD_SMUACK_OFFSET); 460 reg |= bit; 461 writel(reg, mmio + AMD_SMUACK_OFFSET); 462 463 ndev->peer_sta |= bit; 464 } 465 466 static void amd_handle_event(struct amd_ntb_dev *ndev, int vec) 467 { 468 void __iomem *mmio = ndev->self_mmio; 469 u32 status; 470 471 status = readl(mmio + AMD_INTSTAT_OFFSET); 472 if (!(status & AMD_EVENT_INTMASK)) 473 return; 474 475 dev_dbg(ndev_dev(ndev), "status = 0x%x and vec = %d\n", status, vec); 476 477 status &= AMD_EVENT_INTMASK; 478 switch (status) { 479 case AMD_PEER_FLUSH_EVENT: 480 dev_info(ndev_dev(ndev), "Flush is done.\n"); 481 break; 482 case AMD_PEER_RESET_EVENT: 483 amd_ack_smu(ndev, AMD_PEER_RESET_EVENT); 484 485 /* link down first */ 486 ntb_link_event(&ndev->ntb); 487 /* polling peer status */ 488 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 489 490 break; 491 case AMD_PEER_D3_EVENT: 492 case AMD_PEER_PMETO_EVENT: 493 case AMD_LINK_UP_EVENT: 494 case AMD_LINK_DOWN_EVENT: 495 amd_ack_smu(ndev, status); 496 497 /* link down */ 498 ntb_link_event(&ndev->ntb); 499 500 break; 501 case AMD_PEER_D0_EVENT: 502 mmio = ndev->peer_mmio; 503 status = readl(mmio + AMD_PMESTAT_OFFSET); 504 /* check if this is WAKEUP event */ 505 if (status & 0x1) 506 dev_info(ndev_dev(ndev), "Wakeup is done.\n"); 507 508 amd_ack_smu(ndev, AMD_PEER_D0_EVENT); 509 510 /* start a timer to poll link status */ 511 schedule_delayed_work(&ndev->hb_timer, 512 AMD_LINK_HB_TIMEOUT); 513 break; 514 default: 515 dev_info(ndev_dev(ndev), "event status = 0x%x.\n", status); 516 break; 517 } 518 } 519 520 static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec) 521 { 522 dev_dbg(ndev_dev(ndev), "vec %d\n", vec); 523 524 if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1)) 525 amd_handle_event(ndev, vec); 526 527 if (vec < AMD_DB_CNT) 528 ntb_db_event(&ndev->ntb, vec); 529 530 return IRQ_HANDLED; 531 } 532 533 static irqreturn_t ndev_vec_isr(int irq, void *dev) 534 { 535 struct amd_ntb_vec *nvec = dev; 536 537 return ndev_interrupt(nvec->ndev, nvec->num); 538 } 539 540 static irqreturn_t ndev_irq_isr(int irq, void *dev) 541 { 542 struct amd_ntb_dev *ndev = dev; 543 544 return ndev_interrupt(ndev, irq - ndev_pdev(ndev)->irq); 545 } 546 547 static int ndev_init_isr(struct amd_ntb_dev *ndev, 548 int msix_min, int msix_max) 549 { 550 struct pci_dev *pdev; 551 int rc, i, msix_count, node; 552 553 pdev = ndev_pdev(ndev); 554 555 node = dev_to_node(&pdev->dev); 556 557 ndev->db_mask = ndev->db_valid_mask; 558 559 /* Try to set up msix irq */ 560 ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec), 561 GFP_KERNEL, node); 562 if (!ndev->vec) 563 goto err_msix_vec_alloc; 564 565 ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix), 566 GFP_KERNEL, node); 567 if (!ndev->msix) 568 goto err_msix_alloc; 569 570 for (i = 0; i < msix_max; ++i) 571 ndev->msix[i].entry = i; 572 573 msix_count = pci_enable_msix_range(pdev, ndev->msix, 574 msix_min, msix_max); 575 if (msix_count < 0) 576 goto err_msix_enable; 577 578 /* NOTE: Disable MSIX if msix count is less than 16 because of 579 * hardware limitation. 580 */ 581 if (msix_count < msix_min) { 582 pci_disable_msix(pdev); 583 goto err_msix_enable; 584 } 585 586 for (i = 0; i < msix_count; ++i) { 587 ndev->vec[i].ndev = ndev; 588 ndev->vec[i].num = i; 589 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0, 590 "ndev_vec_isr", &ndev->vec[i]); 591 if (rc) 592 goto err_msix_request; 593 } 594 595 dev_dbg(ndev_dev(ndev), "Using msix interrupts\n"); 596 ndev->db_count = msix_min; 597 ndev->msix_vec_count = msix_max; 598 return 0; 599 600 err_msix_request: 601 while (i-- > 0) 602 free_irq(ndev->msix[i].vector, &ndev->vec[i]); 603 pci_disable_msix(pdev); 604 err_msix_enable: 605 kfree(ndev->msix); 606 err_msix_alloc: 607 kfree(ndev->vec); 608 err_msix_vec_alloc: 609 ndev->msix = NULL; 610 ndev->vec = NULL; 611 612 /* Try to set up msi irq */ 613 rc = pci_enable_msi(pdev); 614 if (rc) 615 goto err_msi_enable; 616 617 rc = request_irq(pdev->irq, ndev_irq_isr, 0, 618 "ndev_irq_isr", ndev); 619 if (rc) 620 goto err_msi_request; 621 622 dev_dbg(ndev_dev(ndev), "Using msi interrupts\n"); 623 ndev->db_count = 1; 624 ndev->msix_vec_count = 1; 625 return 0; 626 627 err_msi_request: 628 pci_disable_msi(pdev); 629 err_msi_enable: 630 631 /* Try to set up intx irq */ 632 pci_intx(pdev, 1); 633 634 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED, 635 "ndev_irq_isr", ndev); 636 if (rc) 637 goto err_intx_request; 638 639 dev_dbg(ndev_dev(ndev), "Using intx interrupts\n"); 640 ndev->db_count = 1; 641 ndev->msix_vec_count = 1; 642 return 0; 643 644 err_intx_request: 645 return rc; 646 } 647 648 static void ndev_deinit_isr(struct amd_ntb_dev *ndev) 649 { 650 struct pci_dev *pdev; 651 void __iomem *mmio = ndev->self_mmio; 652 int i; 653 654 pdev = ndev_pdev(ndev); 655 656 /* Mask all doorbell interrupts */ 657 ndev->db_mask = ndev->db_valid_mask; 658 writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET); 659 660 if (ndev->msix) { 661 i = ndev->msix_vec_count; 662 while (i--) 663 free_irq(ndev->msix[i].vector, &ndev->vec[i]); 664 pci_disable_msix(pdev); 665 kfree(ndev->msix); 666 kfree(ndev->vec); 667 } else { 668 free_irq(pdev->irq, ndev); 669 if (pci_dev_msi_enabled(pdev)) 670 pci_disable_msi(pdev); 671 else 672 pci_intx(pdev, 0); 673 } 674 } 675 676 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf, 677 size_t count, loff_t *offp) 678 { 679 struct amd_ntb_dev *ndev; 680 void __iomem *mmio; 681 char *buf; 682 size_t buf_size; 683 ssize_t ret, off; 684 union { u64 v64; u32 v32; u16 v16; } u; 685 686 ndev = filp->private_data; 687 mmio = ndev->self_mmio; 688 689 buf_size = min(count, 0x800ul); 690 691 buf = kmalloc(buf_size, GFP_KERNEL); 692 if (!buf) 693 return -ENOMEM; 694 695 off = 0; 696 697 off += scnprintf(buf + off, buf_size - off, 698 "NTB Device Information:\n"); 699 700 off += scnprintf(buf + off, buf_size - off, 701 "Connection Topology -\t%s\n", 702 ntb_topo_string(ndev->ntb.topo)); 703 704 off += scnprintf(buf + off, buf_size - off, 705 "LNK STA -\t\t%#06x\n", ndev->lnk_sta); 706 707 if (!amd_link_is_up(ndev)) { 708 off += scnprintf(buf + off, buf_size - off, 709 "Link Status -\t\tDown\n"); 710 } else { 711 off += scnprintf(buf + off, buf_size - off, 712 "Link Status -\t\tUp\n"); 713 off += scnprintf(buf + off, buf_size - off, 714 "Link Speed -\t\tPCI-E Gen %u\n", 715 NTB_LNK_STA_SPEED(ndev->lnk_sta)); 716 off += scnprintf(buf + off, buf_size - off, 717 "Link Width -\t\tx%u\n", 718 NTB_LNK_STA_WIDTH(ndev->lnk_sta)); 719 } 720 721 off += scnprintf(buf + off, buf_size - off, 722 "Memory Window Count -\t%u\n", ndev->mw_count); 723 off += scnprintf(buf + off, buf_size - off, 724 "Scratchpad Count -\t%u\n", ndev->spad_count); 725 off += scnprintf(buf + off, buf_size - off, 726 "Doorbell Count -\t%u\n", ndev->db_count); 727 off += scnprintf(buf + off, buf_size - off, 728 "MSIX Vector Count -\t%u\n", ndev->msix_vec_count); 729 730 off += scnprintf(buf + off, buf_size - off, 731 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask); 732 733 u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET); 734 off += scnprintf(buf + off, buf_size - off, 735 "Doorbell Mask -\t\t\t%#06x\n", u.v32); 736 737 u.v32 = readl(mmio + AMD_DBSTAT_OFFSET); 738 off += scnprintf(buf + off, buf_size - off, 739 "Doorbell Bell -\t\t\t%#06x\n", u.v32); 740 741 off += scnprintf(buf + off, buf_size - off, 742 "\nNTB Incoming XLAT:\n"); 743 744 u.v64 = read64(mmio + AMD_BAR1XLAT_OFFSET); 745 off += scnprintf(buf + off, buf_size - off, 746 "XLAT1 -\t\t%#018llx\n", u.v64); 747 748 u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET); 749 off += scnprintf(buf + off, buf_size - off, 750 "XLAT23 -\t\t%#018llx\n", u.v64); 751 752 u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET); 753 off += scnprintf(buf + off, buf_size - off, 754 "XLAT45 -\t\t%#018llx\n", u.v64); 755 756 u.v32 = readl(mmio + AMD_BAR1LMT_OFFSET); 757 off += scnprintf(buf + off, buf_size - off, 758 "LMT1 -\t\t\t%#06x\n", u.v32); 759 760 u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET); 761 off += scnprintf(buf + off, buf_size - off, 762 "LMT23 -\t\t\t%#018llx\n", u.v64); 763 764 u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET); 765 off += scnprintf(buf + off, buf_size - off, 766 "LMT45 -\t\t\t%#018llx\n", u.v64); 767 768 ret = simple_read_from_buffer(ubuf, count, offp, buf, off); 769 kfree(buf); 770 return ret; 771 } 772 773 static void ndev_init_debugfs(struct amd_ntb_dev *ndev) 774 { 775 if (!debugfs_dir) { 776 ndev->debugfs_dir = NULL; 777 ndev->debugfs_info = NULL; 778 } else { 779 ndev->debugfs_dir = 780 debugfs_create_dir(ndev_name(ndev), debugfs_dir); 781 if (!ndev->debugfs_dir) 782 ndev->debugfs_info = NULL; 783 else 784 ndev->debugfs_info = 785 debugfs_create_file("info", S_IRUSR, 786 ndev->debugfs_dir, ndev, 787 &amd_ntb_debugfs_info); 788 } 789 } 790 791 static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev) 792 { 793 debugfs_remove_recursive(ndev->debugfs_dir); 794 } 795 796 static inline void ndev_init_struct(struct amd_ntb_dev *ndev, 797 struct pci_dev *pdev) 798 { 799 ndev->ntb.pdev = pdev; 800 ndev->ntb.topo = NTB_TOPO_NONE; 801 ndev->ntb.ops = &amd_ntb_ops; 802 ndev->int_mask = AMD_EVENT_INTMASK; 803 spin_lock_init(&ndev->db_mask_lock); 804 } 805 806 static int amd_poll_link(struct amd_ntb_dev *ndev) 807 { 808 void __iomem *mmio = ndev->peer_mmio; 809 u32 reg, stat; 810 int rc; 811 812 reg = readl(mmio + AMD_SIDEINFO_OFFSET); 813 reg &= NTB_LIN_STA_ACTIVE_BIT; 814 815 dev_dbg(ndev_dev(ndev), "%s: reg_val = 0x%x.\n", __func__, reg); 816 817 if (reg == ndev->cntl_sta) 818 return 0; 819 820 ndev->cntl_sta = reg; 821 822 rc = pci_read_config_dword(ndev->ntb.pdev, 823 AMD_LINK_STATUS_OFFSET, &stat); 824 if (rc) 825 return 0; 826 ndev->lnk_sta = stat; 827 828 return 1; 829 } 830 831 static void amd_link_hb(struct work_struct *work) 832 { 833 struct amd_ntb_dev *ndev = hb_ndev(work); 834 835 if (amd_poll_link(ndev)) 836 ntb_link_event(&ndev->ntb); 837 838 if (!amd_link_is_up(ndev)) 839 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 840 } 841 842 static int amd_init_isr(struct amd_ntb_dev *ndev) 843 { 844 return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT); 845 } 846 847 static void amd_init_side_info(struct amd_ntb_dev *ndev) 848 { 849 void __iomem *mmio = ndev->self_mmio; 850 unsigned int reg; 851 852 reg = readl(mmio + AMD_SIDEINFO_OFFSET); 853 if (!(reg & AMD_SIDE_READY)) { 854 reg |= AMD_SIDE_READY; 855 writel(reg, mmio + AMD_SIDEINFO_OFFSET); 856 } 857 } 858 859 static void amd_deinit_side_info(struct amd_ntb_dev *ndev) 860 { 861 void __iomem *mmio = ndev->self_mmio; 862 unsigned int reg; 863 864 reg = readl(mmio + AMD_SIDEINFO_OFFSET); 865 if (reg & AMD_SIDE_READY) { 866 reg &= ~AMD_SIDE_READY; 867 writel(reg, mmio + AMD_SIDEINFO_OFFSET); 868 readl(mmio + AMD_SIDEINFO_OFFSET); 869 } 870 } 871 872 static int amd_init_ntb(struct amd_ntb_dev *ndev) 873 { 874 void __iomem *mmio = ndev->self_mmio; 875 876 ndev->mw_count = AMD_MW_CNT; 877 ndev->spad_count = AMD_SPADS_CNT; 878 ndev->db_count = AMD_DB_CNT; 879 880 switch (ndev->ntb.topo) { 881 case NTB_TOPO_PRI: 882 case NTB_TOPO_SEC: 883 ndev->spad_count >>= 1; 884 if (ndev->ntb.topo == NTB_TOPO_PRI) { 885 ndev->self_spad = 0; 886 ndev->peer_spad = 0x20; 887 } else { 888 ndev->self_spad = 0x20; 889 ndev->peer_spad = 0; 890 } 891 892 INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb); 893 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 894 895 break; 896 default: 897 dev_err(ndev_dev(ndev), "AMD NTB does not support B2B mode.\n"); 898 return -EINVAL; 899 } 900 901 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; 902 903 /* Mask event interrupts */ 904 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 905 906 return 0; 907 } 908 909 static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev) 910 { 911 void __iomem *mmio = ndev->self_mmio; 912 u32 info; 913 914 info = readl(mmio + AMD_SIDEINFO_OFFSET); 915 if (info & AMD_SIDE_MASK) 916 return NTB_TOPO_SEC; 917 else 918 return NTB_TOPO_PRI; 919 } 920 921 static int amd_init_dev(struct amd_ntb_dev *ndev) 922 { 923 struct pci_dev *pdev; 924 int rc = 0; 925 926 pdev = ndev_pdev(ndev); 927 928 ndev->ntb.topo = amd_get_topo(ndev); 929 dev_dbg(ndev_dev(ndev), "AMD NTB topo is %s\n", 930 ntb_topo_string(ndev->ntb.topo)); 931 932 rc = amd_init_ntb(ndev); 933 if (rc) 934 return rc; 935 936 rc = amd_init_isr(ndev); 937 if (rc) { 938 dev_err(ndev_dev(ndev), "fail to init isr.\n"); 939 return rc; 940 } 941 942 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; 943 944 return 0; 945 } 946 947 static void amd_deinit_dev(struct amd_ntb_dev *ndev) 948 { 949 cancel_delayed_work_sync(&ndev->hb_timer); 950 951 ndev_deinit_isr(ndev); 952 } 953 954 static int amd_ntb_init_pci(struct amd_ntb_dev *ndev, 955 struct pci_dev *pdev) 956 { 957 int rc; 958 959 pci_set_drvdata(pdev, ndev); 960 961 rc = pci_enable_device(pdev); 962 if (rc) 963 goto err_pci_enable; 964 965 rc = pci_request_regions(pdev, NTB_NAME); 966 if (rc) 967 goto err_pci_regions; 968 969 pci_set_master(pdev); 970 971 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 972 if (rc) { 973 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 974 if (rc) 975 goto err_dma_mask; 976 dev_warn(ndev_dev(ndev), "Cannot DMA highmem\n"); 977 } 978 979 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 980 if (rc) { 981 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 982 if (rc) 983 goto err_dma_mask; 984 dev_warn(ndev_dev(ndev), "Cannot DMA consistent highmem\n"); 985 } 986 987 ndev->self_mmio = pci_iomap(pdev, 0, 0); 988 if (!ndev->self_mmio) { 989 rc = -EIO; 990 goto err_dma_mask; 991 } 992 ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET; 993 994 return 0; 995 996 err_dma_mask: 997 pci_clear_master(pdev); 998 err_pci_regions: 999 pci_disable_device(pdev); 1000 err_pci_enable: 1001 pci_set_drvdata(pdev, NULL); 1002 return rc; 1003 } 1004 1005 static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev) 1006 { 1007 struct pci_dev *pdev = ndev_pdev(ndev); 1008 1009 pci_iounmap(pdev, ndev->self_mmio); 1010 1011 pci_clear_master(pdev); 1012 pci_release_regions(pdev); 1013 pci_disable_device(pdev); 1014 pci_set_drvdata(pdev, NULL); 1015 } 1016 1017 static int amd_ntb_pci_probe(struct pci_dev *pdev, 1018 const struct pci_device_id *id) 1019 { 1020 struct amd_ntb_dev *ndev; 1021 int rc, node; 1022 1023 node = dev_to_node(&pdev->dev); 1024 1025 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node); 1026 if (!ndev) { 1027 rc = -ENOMEM; 1028 goto err_ndev; 1029 } 1030 1031 ndev_init_struct(ndev, pdev); 1032 1033 rc = amd_ntb_init_pci(ndev, pdev); 1034 if (rc) 1035 goto err_init_pci; 1036 1037 rc = amd_init_dev(ndev); 1038 if (rc) 1039 goto err_init_dev; 1040 1041 /* write side info */ 1042 amd_init_side_info(ndev); 1043 1044 amd_poll_link(ndev); 1045 1046 ndev_init_debugfs(ndev); 1047 1048 rc = ntb_register_device(&ndev->ntb); 1049 if (rc) 1050 goto err_register; 1051 1052 dev_info(&pdev->dev, "NTB device registered.\n"); 1053 1054 return 0; 1055 1056 err_register: 1057 ndev_deinit_debugfs(ndev); 1058 amd_deinit_dev(ndev); 1059 err_init_dev: 1060 amd_ntb_deinit_pci(ndev); 1061 err_init_pci: 1062 kfree(ndev); 1063 err_ndev: 1064 return rc; 1065 } 1066 1067 static void amd_ntb_pci_remove(struct pci_dev *pdev) 1068 { 1069 struct amd_ntb_dev *ndev = pci_get_drvdata(pdev); 1070 1071 ntb_unregister_device(&ndev->ntb); 1072 ndev_deinit_debugfs(ndev); 1073 amd_deinit_side_info(ndev); 1074 amd_deinit_dev(ndev); 1075 amd_ntb_deinit_pci(ndev); 1076 kfree(ndev); 1077 } 1078 1079 static const struct file_operations amd_ntb_debugfs_info = { 1080 .owner = THIS_MODULE, 1081 .open = simple_open, 1082 .read = ndev_debugfs_read, 1083 }; 1084 1085 static const struct pci_device_id amd_ntb_pci_tbl[] = { 1086 {PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_NTB)}, 1087 {0} 1088 }; 1089 MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl); 1090 1091 static struct pci_driver amd_ntb_pci_driver = { 1092 .name = KBUILD_MODNAME, 1093 .id_table = amd_ntb_pci_tbl, 1094 .probe = amd_ntb_pci_probe, 1095 .remove = amd_ntb_pci_remove, 1096 }; 1097 1098 static int __init amd_ntb_pci_driver_init(void) 1099 { 1100 pr_info("%s %s\n", NTB_DESC, NTB_VER); 1101 1102 if (debugfs_initialized()) 1103 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 1104 1105 return pci_register_driver(&amd_ntb_pci_driver); 1106 } 1107 module_init(amd_ntb_pci_driver_init); 1108 1109 static void __exit amd_ntb_pci_driver_exit(void) 1110 { 1111 pci_unregister_driver(&amd_ntb_pci_driver); 1112 debugfs_remove_recursive(debugfs_dir); 1113 } 1114 module_exit(amd_ntb_pci_driver_exit); 1115