1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Thunderbolt driver - NHI driver 4 * 5 * The NHI (native host interface) is the pci device that allows us to send and 6 * receive frames from the thunderbolt bus. 7 * 8 * Copyright (c) 2014 Andreas Noever <andreas.noever@gmail.com> 9 * Copyright (C) 2018, Intel Corporation 10 */ 11 12 #include <linux/pm_runtime.h> 13 #include <linux/slab.h> 14 #include <linux/errno.h> 15 #include <linux/pci.h> 16 #include <linux/dma-mapping.h> 17 #include <linux/interrupt.h> 18 #include <linux/iommu.h> 19 #include <linux/module.h> 20 #include <linux/delay.h> 21 #include <linux/property.h> 22 #include <linux/string_helpers.h> 23 24 #include "nhi.h" 25 #include "nhi_regs.h" 26 #include "tb.h" 27 28 #define RING_TYPE(ring) ((ring)->is_tx ? "TX ring" : "RX ring") 29 30 #define RING_FIRST_USABLE_HOPID 1 31 32 /* 33 * Minimal number of vectors when we use MSI-X. Two for control channel 34 * Rx/Tx and the rest four are for cross domain DMA paths. 35 */ 36 #define MSIX_MIN_VECS 6 37 #define MSIX_MAX_VECS 16 38 39 #define NHI_MAILBOX_TIMEOUT 500 /* ms */ 40 41 #define QUIRK_AUTO_CLEAR_INT BIT(0) 42 43 static int ring_interrupt_index(struct tb_ring *ring) 44 { 45 int bit = ring->hop; 46 if (!ring->is_tx) 47 bit += ring->nhi->hop_count; 48 return bit; 49 } 50 51 /* 52 * ring_interrupt_active() - activate/deactivate interrupts for a single ring 53 * 54 * ring->nhi->lock must be held. 55 */ 56 static void ring_interrupt_active(struct tb_ring *ring, bool active) 57 { 58 int reg = REG_RING_INTERRUPT_BASE + 59 ring_interrupt_index(ring) / 32 * 4; 60 int bit = ring_interrupt_index(ring) & 31; 61 int mask = 1 << bit; 62 u32 old, new; 63 64 if (ring->irq > 0) { 65 u32 step, shift, ivr, misc; 66 void __iomem *ivr_base; 67 int index; 68 69 if (ring->is_tx) 70 index = ring->hop; 71 else 72 index = ring->hop + ring->nhi->hop_count; 73 74 if (ring->nhi->quirks & QUIRK_AUTO_CLEAR_INT) { 75 /* 76 * Ask the hardware to clear interrupt status 77 * bits automatically since we already know 78 * which interrupt was triggered. 79 */ 80 misc = ioread32(ring->nhi->iobase + REG_DMA_MISC); 81 if (!(misc & REG_DMA_MISC_INT_AUTO_CLEAR)) { 82 misc |= REG_DMA_MISC_INT_AUTO_CLEAR; 83 iowrite32(misc, ring->nhi->iobase + REG_DMA_MISC); 84 } 85 } 86 87 ivr_base = ring->nhi->iobase + REG_INT_VEC_ALLOC_BASE; 88 step = index / REG_INT_VEC_ALLOC_REGS * REG_INT_VEC_ALLOC_BITS; 89 shift = index % REG_INT_VEC_ALLOC_REGS * REG_INT_VEC_ALLOC_BITS; 90 ivr = ioread32(ivr_base + step); 91 ivr &= ~(REG_INT_VEC_ALLOC_MASK << shift); 92 if (active) 93 ivr |= ring->vector << shift; 94 iowrite32(ivr, ivr_base + step); 95 } 96 97 old = ioread32(ring->nhi->iobase + reg); 98 if (active) 99 new = old | mask; 100 else 101 new = old & ~mask; 102 103 dev_dbg(&ring->nhi->pdev->dev, 104 "%s interrupt at register %#x bit %d (%#x -> %#x)\n", 105 active ? "enabling" : "disabling", reg, bit, old, new); 106 107 if (new == old) 108 dev_WARN(&ring->nhi->pdev->dev, 109 "interrupt for %s %d is already %s\n", 110 RING_TYPE(ring), ring->hop, 111 active ? "enabled" : "disabled"); 112 iowrite32(new, ring->nhi->iobase + reg); 113 } 114 115 /* 116 * nhi_disable_interrupts() - disable interrupts for all rings 117 * 118 * Use only during init and shutdown. 119 */ 120 static void nhi_disable_interrupts(struct tb_nhi *nhi) 121 { 122 int i = 0; 123 /* disable interrupts */ 124 for (i = 0; i < RING_INTERRUPT_REG_COUNT(nhi); i++) 125 iowrite32(0, nhi->iobase + REG_RING_INTERRUPT_BASE + 4 * i); 126 127 /* clear interrupt status bits */ 128 for (i = 0; i < RING_NOTIFY_REG_COUNT(nhi); i++) 129 ioread32(nhi->iobase + REG_RING_NOTIFY_BASE + 4 * i); 130 } 131 132 /* ring helper methods */ 133 134 static void __iomem *ring_desc_base(struct tb_ring *ring) 135 { 136 void __iomem *io = ring->nhi->iobase; 137 io += ring->is_tx ? REG_TX_RING_BASE : REG_RX_RING_BASE; 138 io += ring->hop * 16; 139 return io; 140 } 141 142 static void __iomem *ring_options_base(struct tb_ring *ring) 143 { 144 void __iomem *io = ring->nhi->iobase; 145 io += ring->is_tx ? REG_TX_OPTIONS_BASE : REG_RX_OPTIONS_BASE; 146 io += ring->hop * 32; 147 return io; 148 } 149 150 static void ring_iowrite_cons(struct tb_ring *ring, u16 cons) 151 { 152 /* 153 * The other 16-bits in the register is read-only and writes to it 154 * are ignored by the hardware so we can save one ioread32() by 155 * filling the read-only bits with zeroes. 156 */ 157 iowrite32(cons, ring_desc_base(ring) + 8); 158 } 159 160 static void ring_iowrite_prod(struct tb_ring *ring, u16 prod) 161 { 162 /* See ring_iowrite_cons() above for explanation */ 163 iowrite32(prod << 16, ring_desc_base(ring) + 8); 164 } 165 166 static void ring_iowrite32desc(struct tb_ring *ring, u32 value, u32 offset) 167 { 168 iowrite32(value, ring_desc_base(ring) + offset); 169 } 170 171 static void ring_iowrite64desc(struct tb_ring *ring, u64 value, u32 offset) 172 { 173 iowrite32(value, ring_desc_base(ring) + offset); 174 iowrite32(value >> 32, ring_desc_base(ring) + offset + 4); 175 } 176 177 static void ring_iowrite32options(struct tb_ring *ring, u32 value, u32 offset) 178 { 179 iowrite32(value, ring_options_base(ring) + offset); 180 } 181 182 static bool ring_full(struct tb_ring *ring) 183 { 184 return ((ring->head + 1) % ring->size) == ring->tail; 185 } 186 187 static bool ring_empty(struct tb_ring *ring) 188 { 189 return ring->head == ring->tail; 190 } 191 192 /* 193 * ring_write_descriptors() - post frames from ring->queue to the controller 194 * 195 * ring->lock is held. 196 */ 197 static void ring_write_descriptors(struct tb_ring *ring) 198 { 199 struct ring_frame *frame, *n; 200 struct ring_desc *descriptor; 201 list_for_each_entry_safe(frame, n, &ring->queue, list) { 202 if (ring_full(ring)) 203 break; 204 list_move_tail(&frame->list, &ring->in_flight); 205 descriptor = &ring->descriptors[ring->head]; 206 descriptor->phys = frame->buffer_phy; 207 descriptor->time = 0; 208 descriptor->flags = RING_DESC_POSTED | RING_DESC_INTERRUPT; 209 if (ring->is_tx) { 210 descriptor->length = frame->size; 211 descriptor->eof = frame->eof; 212 descriptor->sof = frame->sof; 213 } 214 ring->head = (ring->head + 1) % ring->size; 215 if (ring->is_tx) 216 ring_iowrite_prod(ring, ring->head); 217 else 218 ring_iowrite_cons(ring, ring->head); 219 } 220 } 221 222 /* 223 * ring_work() - progress completed frames 224 * 225 * If the ring is shutting down then all frames are marked as canceled and 226 * their callbacks are invoked. 227 * 228 * Otherwise we collect all completed frame from the ring buffer, write new 229 * frame to the ring buffer and invoke the callbacks for the completed frames. 230 */ 231 static void ring_work(struct work_struct *work) 232 { 233 struct tb_ring *ring = container_of(work, typeof(*ring), work); 234 struct ring_frame *frame; 235 bool canceled = false; 236 unsigned long flags; 237 LIST_HEAD(done); 238 239 spin_lock_irqsave(&ring->lock, flags); 240 241 if (!ring->running) { 242 /* Move all frames to done and mark them as canceled. */ 243 list_splice_tail_init(&ring->in_flight, &done); 244 list_splice_tail_init(&ring->queue, &done); 245 canceled = true; 246 goto invoke_callback; 247 } 248 249 while (!ring_empty(ring)) { 250 if (!(ring->descriptors[ring->tail].flags 251 & RING_DESC_COMPLETED)) 252 break; 253 frame = list_first_entry(&ring->in_flight, typeof(*frame), 254 list); 255 list_move_tail(&frame->list, &done); 256 if (!ring->is_tx) { 257 frame->size = ring->descriptors[ring->tail].length; 258 frame->eof = ring->descriptors[ring->tail].eof; 259 frame->sof = ring->descriptors[ring->tail].sof; 260 frame->flags = ring->descriptors[ring->tail].flags; 261 } 262 ring->tail = (ring->tail + 1) % ring->size; 263 } 264 ring_write_descriptors(ring); 265 266 invoke_callback: 267 /* allow callbacks to schedule new work */ 268 spin_unlock_irqrestore(&ring->lock, flags); 269 while (!list_empty(&done)) { 270 frame = list_first_entry(&done, typeof(*frame), list); 271 /* 272 * The callback may reenqueue or delete frame. 273 * Do not hold on to it. 274 */ 275 list_del_init(&frame->list); 276 if (frame->callback) 277 frame->callback(ring, frame, canceled); 278 } 279 } 280 281 int __tb_ring_enqueue(struct tb_ring *ring, struct ring_frame *frame) 282 { 283 unsigned long flags; 284 int ret = 0; 285 286 spin_lock_irqsave(&ring->lock, flags); 287 if (ring->running) { 288 list_add_tail(&frame->list, &ring->queue); 289 ring_write_descriptors(ring); 290 } else { 291 ret = -ESHUTDOWN; 292 } 293 spin_unlock_irqrestore(&ring->lock, flags); 294 return ret; 295 } 296 EXPORT_SYMBOL_GPL(__tb_ring_enqueue); 297 298 /** 299 * tb_ring_poll() - Poll one completed frame from the ring 300 * @ring: Ring to poll 301 * 302 * This function can be called when @start_poll callback of the @ring 303 * has been called. It will read one completed frame from the ring and 304 * return it to the caller. Returns %NULL if there is no more completed 305 * frames. 306 */ 307 struct ring_frame *tb_ring_poll(struct tb_ring *ring) 308 { 309 struct ring_frame *frame = NULL; 310 unsigned long flags; 311 312 spin_lock_irqsave(&ring->lock, flags); 313 if (!ring->running) 314 goto unlock; 315 if (ring_empty(ring)) 316 goto unlock; 317 318 if (ring->descriptors[ring->tail].flags & RING_DESC_COMPLETED) { 319 frame = list_first_entry(&ring->in_flight, typeof(*frame), 320 list); 321 list_del_init(&frame->list); 322 323 if (!ring->is_tx) { 324 frame->size = ring->descriptors[ring->tail].length; 325 frame->eof = ring->descriptors[ring->tail].eof; 326 frame->sof = ring->descriptors[ring->tail].sof; 327 frame->flags = ring->descriptors[ring->tail].flags; 328 } 329 330 ring->tail = (ring->tail + 1) % ring->size; 331 } 332 333 unlock: 334 spin_unlock_irqrestore(&ring->lock, flags); 335 return frame; 336 } 337 EXPORT_SYMBOL_GPL(tb_ring_poll); 338 339 static void __ring_interrupt_mask(struct tb_ring *ring, bool mask) 340 { 341 int idx = ring_interrupt_index(ring); 342 int reg = REG_RING_INTERRUPT_BASE + idx / 32 * 4; 343 int bit = idx % 32; 344 u32 val; 345 346 val = ioread32(ring->nhi->iobase + reg); 347 if (mask) 348 val &= ~BIT(bit); 349 else 350 val |= BIT(bit); 351 iowrite32(val, ring->nhi->iobase + reg); 352 } 353 354 /* Both @nhi->lock and @ring->lock should be held */ 355 static void __ring_interrupt(struct tb_ring *ring) 356 { 357 if (!ring->running) 358 return; 359 360 if (ring->start_poll) { 361 __ring_interrupt_mask(ring, true); 362 ring->start_poll(ring->poll_data); 363 } else { 364 schedule_work(&ring->work); 365 } 366 } 367 368 /** 369 * tb_ring_poll_complete() - Re-start interrupt for the ring 370 * @ring: Ring to re-start the interrupt 371 * 372 * This will re-start (unmask) the ring interrupt once the user is done 373 * with polling. 374 */ 375 void tb_ring_poll_complete(struct tb_ring *ring) 376 { 377 unsigned long flags; 378 379 spin_lock_irqsave(&ring->nhi->lock, flags); 380 spin_lock(&ring->lock); 381 if (ring->start_poll) 382 __ring_interrupt_mask(ring, false); 383 spin_unlock(&ring->lock); 384 spin_unlock_irqrestore(&ring->nhi->lock, flags); 385 } 386 EXPORT_SYMBOL_GPL(tb_ring_poll_complete); 387 388 static void ring_clear_msix(const struct tb_ring *ring) 389 { 390 if (ring->nhi->quirks & QUIRK_AUTO_CLEAR_INT) 391 return; 392 393 if (ring->is_tx) 394 ioread32(ring->nhi->iobase + REG_RING_NOTIFY_BASE); 395 else 396 ioread32(ring->nhi->iobase + REG_RING_NOTIFY_BASE + 397 4 * (ring->nhi->hop_count / 32)); 398 } 399 400 static irqreturn_t ring_msix(int irq, void *data) 401 { 402 struct tb_ring *ring = data; 403 404 spin_lock(&ring->nhi->lock); 405 ring_clear_msix(ring); 406 spin_lock(&ring->lock); 407 __ring_interrupt(ring); 408 spin_unlock(&ring->lock); 409 spin_unlock(&ring->nhi->lock); 410 411 return IRQ_HANDLED; 412 } 413 414 static int ring_request_msix(struct tb_ring *ring, bool no_suspend) 415 { 416 struct tb_nhi *nhi = ring->nhi; 417 unsigned long irqflags; 418 int ret; 419 420 if (!nhi->pdev->msix_enabled) 421 return 0; 422 423 ret = ida_simple_get(&nhi->msix_ida, 0, MSIX_MAX_VECS, GFP_KERNEL); 424 if (ret < 0) 425 return ret; 426 427 ring->vector = ret; 428 429 ret = pci_irq_vector(ring->nhi->pdev, ring->vector); 430 if (ret < 0) 431 goto err_ida_remove; 432 433 ring->irq = ret; 434 435 irqflags = no_suspend ? IRQF_NO_SUSPEND : 0; 436 ret = request_irq(ring->irq, ring_msix, irqflags, "thunderbolt", ring); 437 if (ret) 438 goto err_ida_remove; 439 440 return 0; 441 442 err_ida_remove: 443 ida_simple_remove(&nhi->msix_ida, ring->vector); 444 445 return ret; 446 } 447 448 static void ring_release_msix(struct tb_ring *ring) 449 { 450 if (ring->irq <= 0) 451 return; 452 453 free_irq(ring->irq, ring); 454 ida_simple_remove(&ring->nhi->msix_ida, ring->vector); 455 ring->vector = 0; 456 ring->irq = 0; 457 } 458 459 static int nhi_alloc_hop(struct tb_nhi *nhi, struct tb_ring *ring) 460 { 461 int ret = 0; 462 463 spin_lock_irq(&nhi->lock); 464 465 if (ring->hop < 0) { 466 unsigned int i; 467 468 /* 469 * Automatically allocate HopID from the non-reserved 470 * range 1 .. hop_count - 1. 471 */ 472 for (i = RING_FIRST_USABLE_HOPID; i < nhi->hop_count; i++) { 473 if (ring->is_tx) { 474 if (!nhi->tx_rings[i]) { 475 ring->hop = i; 476 break; 477 } 478 } else { 479 if (!nhi->rx_rings[i]) { 480 ring->hop = i; 481 break; 482 } 483 } 484 } 485 } 486 487 if (ring->hop < 0 || ring->hop >= nhi->hop_count) { 488 dev_warn(&nhi->pdev->dev, "invalid hop: %d\n", ring->hop); 489 ret = -EINVAL; 490 goto err_unlock; 491 } 492 if (ring->is_tx && nhi->tx_rings[ring->hop]) { 493 dev_warn(&nhi->pdev->dev, "TX hop %d already allocated\n", 494 ring->hop); 495 ret = -EBUSY; 496 goto err_unlock; 497 } else if (!ring->is_tx && nhi->rx_rings[ring->hop]) { 498 dev_warn(&nhi->pdev->dev, "RX hop %d already allocated\n", 499 ring->hop); 500 ret = -EBUSY; 501 goto err_unlock; 502 } 503 504 if (ring->is_tx) 505 nhi->tx_rings[ring->hop] = ring; 506 else 507 nhi->rx_rings[ring->hop] = ring; 508 509 err_unlock: 510 spin_unlock_irq(&nhi->lock); 511 512 return ret; 513 } 514 515 static struct tb_ring *tb_ring_alloc(struct tb_nhi *nhi, u32 hop, int size, 516 bool transmit, unsigned int flags, 517 int e2e_tx_hop, u16 sof_mask, u16 eof_mask, 518 void (*start_poll)(void *), 519 void *poll_data) 520 { 521 struct tb_ring *ring = NULL; 522 523 dev_dbg(&nhi->pdev->dev, "allocating %s ring %d of size %d\n", 524 transmit ? "TX" : "RX", hop, size); 525 526 ring = kzalloc(sizeof(*ring), GFP_KERNEL); 527 if (!ring) 528 return NULL; 529 530 spin_lock_init(&ring->lock); 531 INIT_LIST_HEAD(&ring->queue); 532 INIT_LIST_HEAD(&ring->in_flight); 533 INIT_WORK(&ring->work, ring_work); 534 535 ring->nhi = nhi; 536 ring->hop = hop; 537 ring->is_tx = transmit; 538 ring->size = size; 539 ring->flags = flags; 540 ring->e2e_tx_hop = e2e_tx_hop; 541 ring->sof_mask = sof_mask; 542 ring->eof_mask = eof_mask; 543 ring->head = 0; 544 ring->tail = 0; 545 ring->running = false; 546 ring->start_poll = start_poll; 547 ring->poll_data = poll_data; 548 549 ring->descriptors = dma_alloc_coherent(&ring->nhi->pdev->dev, 550 size * sizeof(*ring->descriptors), 551 &ring->descriptors_dma, GFP_KERNEL | __GFP_ZERO); 552 if (!ring->descriptors) 553 goto err_free_ring; 554 555 if (ring_request_msix(ring, flags & RING_FLAG_NO_SUSPEND)) 556 goto err_free_descs; 557 558 if (nhi_alloc_hop(nhi, ring)) 559 goto err_release_msix; 560 561 return ring; 562 563 err_release_msix: 564 ring_release_msix(ring); 565 err_free_descs: 566 dma_free_coherent(&ring->nhi->pdev->dev, 567 ring->size * sizeof(*ring->descriptors), 568 ring->descriptors, ring->descriptors_dma); 569 err_free_ring: 570 kfree(ring); 571 572 return NULL; 573 } 574 575 /** 576 * tb_ring_alloc_tx() - Allocate DMA ring for transmit 577 * @nhi: Pointer to the NHI the ring is to be allocated 578 * @hop: HopID (ring) to allocate 579 * @size: Number of entries in the ring 580 * @flags: Flags for the ring 581 */ 582 struct tb_ring *tb_ring_alloc_tx(struct tb_nhi *nhi, int hop, int size, 583 unsigned int flags) 584 { 585 return tb_ring_alloc(nhi, hop, size, true, flags, 0, 0, 0, NULL, NULL); 586 } 587 EXPORT_SYMBOL_GPL(tb_ring_alloc_tx); 588 589 /** 590 * tb_ring_alloc_rx() - Allocate DMA ring for receive 591 * @nhi: Pointer to the NHI the ring is to be allocated 592 * @hop: HopID (ring) to allocate. Pass %-1 for automatic allocation. 593 * @size: Number of entries in the ring 594 * @flags: Flags for the ring 595 * @e2e_tx_hop: Transmit HopID when E2E is enabled in @flags 596 * @sof_mask: Mask of PDF values that start a frame 597 * @eof_mask: Mask of PDF values that end a frame 598 * @start_poll: If not %NULL the ring will call this function when an 599 * interrupt is triggered and masked, instead of callback 600 * in each Rx frame. 601 * @poll_data: Optional data passed to @start_poll 602 */ 603 struct tb_ring *tb_ring_alloc_rx(struct tb_nhi *nhi, int hop, int size, 604 unsigned int flags, int e2e_tx_hop, 605 u16 sof_mask, u16 eof_mask, 606 void (*start_poll)(void *), void *poll_data) 607 { 608 return tb_ring_alloc(nhi, hop, size, false, flags, e2e_tx_hop, sof_mask, eof_mask, 609 start_poll, poll_data); 610 } 611 EXPORT_SYMBOL_GPL(tb_ring_alloc_rx); 612 613 /** 614 * tb_ring_start() - enable a ring 615 * @ring: Ring to start 616 * 617 * Must not be invoked in parallel with tb_ring_stop(). 618 */ 619 void tb_ring_start(struct tb_ring *ring) 620 { 621 u16 frame_size; 622 u32 flags; 623 624 spin_lock_irq(&ring->nhi->lock); 625 spin_lock(&ring->lock); 626 if (ring->nhi->going_away) 627 goto err; 628 if (ring->running) { 629 dev_WARN(&ring->nhi->pdev->dev, "ring already started\n"); 630 goto err; 631 } 632 dev_dbg(&ring->nhi->pdev->dev, "starting %s %d\n", 633 RING_TYPE(ring), ring->hop); 634 635 if (ring->flags & RING_FLAG_FRAME) { 636 /* Means 4096 */ 637 frame_size = 0; 638 flags = RING_FLAG_ENABLE; 639 } else { 640 frame_size = TB_FRAME_SIZE; 641 flags = RING_FLAG_ENABLE | RING_FLAG_RAW; 642 } 643 644 ring_iowrite64desc(ring, ring->descriptors_dma, 0); 645 if (ring->is_tx) { 646 ring_iowrite32desc(ring, ring->size, 12); 647 ring_iowrite32options(ring, 0, 4); /* time releated ? */ 648 ring_iowrite32options(ring, flags, 0); 649 } else { 650 u32 sof_eof_mask = ring->sof_mask << 16 | ring->eof_mask; 651 652 ring_iowrite32desc(ring, (frame_size << 16) | ring->size, 12); 653 ring_iowrite32options(ring, sof_eof_mask, 4); 654 ring_iowrite32options(ring, flags, 0); 655 } 656 657 /* 658 * Now that the ring valid bit is set we can configure E2E if 659 * enabled for the ring. 660 */ 661 if (ring->flags & RING_FLAG_E2E) { 662 if (!ring->is_tx) { 663 u32 hop; 664 665 hop = ring->e2e_tx_hop << REG_RX_OPTIONS_E2E_HOP_SHIFT; 666 hop &= REG_RX_OPTIONS_E2E_HOP_MASK; 667 flags |= hop; 668 669 dev_dbg(&ring->nhi->pdev->dev, 670 "enabling E2E for %s %d with TX HopID %d\n", 671 RING_TYPE(ring), ring->hop, ring->e2e_tx_hop); 672 } else { 673 dev_dbg(&ring->nhi->pdev->dev, "enabling E2E for %s %d\n", 674 RING_TYPE(ring), ring->hop); 675 } 676 677 flags |= RING_FLAG_E2E_FLOW_CONTROL; 678 ring_iowrite32options(ring, flags, 0); 679 } 680 681 ring_interrupt_active(ring, true); 682 ring->running = true; 683 err: 684 spin_unlock(&ring->lock); 685 spin_unlock_irq(&ring->nhi->lock); 686 } 687 EXPORT_SYMBOL_GPL(tb_ring_start); 688 689 /** 690 * tb_ring_stop() - shutdown a ring 691 * @ring: Ring to stop 692 * 693 * Must not be invoked from a callback. 694 * 695 * This method will disable the ring. Further calls to 696 * tb_ring_tx/tb_ring_rx will return -ESHUTDOWN until ring_stop has been 697 * called. 698 * 699 * All enqueued frames will be canceled and their callbacks will be executed 700 * with frame->canceled set to true (on the callback thread). This method 701 * returns only after all callback invocations have finished. 702 */ 703 void tb_ring_stop(struct tb_ring *ring) 704 { 705 spin_lock_irq(&ring->nhi->lock); 706 spin_lock(&ring->lock); 707 dev_dbg(&ring->nhi->pdev->dev, "stopping %s %d\n", 708 RING_TYPE(ring), ring->hop); 709 if (ring->nhi->going_away) 710 goto err; 711 if (!ring->running) { 712 dev_WARN(&ring->nhi->pdev->dev, "%s %d already stopped\n", 713 RING_TYPE(ring), ring->hop); 714 goto err; 715 } 716 ring_interrupt_active(ring, false); 717 718 ring_iowrite32options(ring, 0, 0); 719 ring_iowrite64desc(ring, 0, 0); 720 ring_iowrite32desc(ring, 0, 8); 721 ring_iowrite32desc(ring, 0, 12); 722 ring->head = 0; 723 ring->tail = 0; 724 ring->running = false; 725 726 err: 727 spin_unlock(&ring->lock); 728 spin_unlock_irq(&ring->nhi->lock); 729 730 /* 731 * schedule ring->work to invoke callbacks on all remaining frames. 732 */ 733 schedule_work(&ring->work); 734 flush_work(&ring->work); 735 } 736 EXPORT_SYMBOL_GPL(tb_ring_stop); 737 738 /* 739 * tb_ring_free() - free ring 740 * 741 * When this method returns all invocations of ring->callback will have 742 * finished. 743 * 744 * Ring must be stopped. 745 * 746 * Must NOT be called from ring_frame->callback! 747 */ 748 void tb_ring_free(struct tb_ring *ring) 749 { 750 spin_lock_irq(&ring->nhi->lock); 751 /* 752 * Dissociate the ring from the NHI. This also ensures that 753 * nhi_interrupt_work cannot reschedule ring->work. 754 */ 755 if (ring->is_tx) 756 ring->nhi->tx_rings[ring->hop] = NULL; 757 else 758 ring->nhi->rx_rings[ring->hop] = NULL; 759 760 if (ring->running) { 761 dev_WARN(&ring->nhi->pdev->dev, "%s %d still running\n", 762 RING_TYPE(ring), ring->hop); 763 } 764 spin_unlock_irq(&ring->nhi->lock); 765 766 ring_release_msix(ring); 767 768 dma_free_coherent(&ring->nhi->pdev->dev, 769 ring->size * sizeof(*ring->descriptors), 770 ring->descriptors, ring->descriptors_dma); 771 772 ring->descriptors = NULL; 773 ring->descriptors_dma = 0; 774 775 776 dev_dbg(&ring->nhi->pdev->dev, "freeing %s %d\n", RING_TYPE(ring), 777 ring->hop); 778 779 /* 780 * ring->work can no longer be scheduled (it is scheduled only 781 * by nhi_interrupt_work, ring_stop and ring_msix). Wait for it 782 * to finish before freeing the ring. 783 */ 784 flush_work(&ring->work); 785 kfree(ring); 786 } 787 EXPORT_SYMBOL_GPL(tb_ring_free); 788 789 /** 790 * nhi_mailbox_cmd() - Send a command through NHI mailbox 791 * @nhi: Pointer to the NHI structure 792 * @cmd: Command to send 793 * @data: Data to be send with the command 794 * 795 * Sends mailbox command to the firmware running on NHI. Returns %0 in 796 * case of success and negative errno in case of failure. 797 */ 798 int nhi_mailbox_cmd(struct tb_nhi *nhi, enum nhi_mailbox_cmd cmd, u32 data) 799 { 800 ktime_t timeout; 801 u32 val; 802 803 iowrite32(data, nhi->iobase + REG_INMAIL_DATA); 804 805 val = ioread32(nhi->iobase + REG_INMAIL_CMD); 806 val &= ~(REG_INMAIL_CMD_MASK | REG_INMAIL_ERROR); 807 val |= REG_INMAIL_OP_REQUEST | cmd; 808 iowrite32(val, nhi->iobase + REG_INMAIL_CMD); 809 810 timeout = ktime_add_ms(ktime_get(), NHI_MAILBOX_TIMEOUT); 811 do { 812 val = ioread32(nhi->iobase + REG_INMAIL_CMD); 813 if (!(val & REG_INMAIL_OP_REQUEST)) 814 break; 815 usleep_range(10, 20); 816 } while (ktime_before(ktime_get(), timeout)); 817 818 if (val & REG_INMAIL_OP_REQUEST) 819 return -ETIMEDOUT; 820 if (val & REG_INMAIL_ERROR) 821 return -EIO; 822 823 return 0; 824 } 825 826 /** 827 * nhi_mailbox_mode() - Return current firmware operation mode 828 * @nhi: Pointer to the NHI structure 829 * 830 * The function reads current firmware operation mode using NHI mailbox 831 * registers and returns it to the caller. 832 */ 833 enum nhi_fw_mode nhi_mailbox_mode(struct tb_nhi *nhi) 834 { 835 u32 val; 836 837 val = ioread32(nhi->iobase + REG_OUTMAIL_CMD); 838 val &= REG_OUTMAIL_CMD_OPMODE_MASK; 839 val >>= REG_OUTMAIL_CMD_OPMODE_SHIFT; 840 841 return (enum nhi_fw_mode)val; 842 } 843 844 static void nhi_interrupt_work(struct work_struct *work) 845 { 846 struct tb_nhi *nhi = container_of(work, typeof(*nhi), interrupt_work); 847 int value = 0; /* Suppress uninitialized usage warning. */ 848 int bit; 849 int hop = -1; 850 int type = 0; /* current interrupt type 0: TX, 1: RX, 2: RX overflow */ 851 struct tb_ring *ring; 852 853 spin_lock_irq(&nhi->lock); 854 855 /* 856 * Starting at REG_RING_NOTIFY_BASE there are three status bitfields 857 * (TX, RX, RX overflow). We iterate over the bits and read a new 858 * dwords as required. The registers are cleared on read. 859 */ 860 for (bit = 0; bit < 3 * nhi->hop_count; bit++) { 861 if (bit % 32 == 0) 862 value = ioread32(nhi->iobase 863 + REG_RING_NOTIFY_BASE 864 + 4 * (bit / 32)); 865 if (++hop == nhi->hop_count) { 866 hop = 0; 867 type++; 868 } 869 if ((value & (1 << (bit % 32))) == 0) 870 continue; 871 if (type == 2) { 872 dev_warn(&nhi->pdev->dev, 873 "RX overflow for ring %d\n", 874 hop); 875 continue; 876 } 877 if (type == 0) 878 ring = nhi->tx_rings[hop]; 879 else 880 ring = nhi->rx_rings[hop]; 881 if (ring == NULL) { 882 dev_warn(&nhi->pdev->dev, 883 "got interrupt for inactive %s ring %d\n", 884 type ? "RX" : "TX", 885 hop); 886 continue; 887 } 888 889 spin_lock(&ring->lock); 890 __ring_interrupt(ring); 891 spin_unlock(&ring->lock); 892 } 893 spin_unlock_irq(&nhi->lock); 894 } 895 896 static irqreturn_t nhi_msi(int irq, void *data) 897 { 898 struct tb_nhi *nhi = data; 899 schedule_work(&nhi->interrupt_work); 900 return IRQ_HANDLED; 901 } 902 903 static int __nhi_suspend_noirq(struct device *dev, bool wakeup) 904 { 905 struct pci_dev *pdev = to_pci_dev(dev); 906 struct tb *tb = pci_get_drvdata(pdev); 907 struct tb_nhi *nhi = tb->nhi; 908 int ret; 909 910 ret = tb_domain_suspend_noirq(tb); 911 if (ret) 912 return ret; 913 914 if (nhi->ops && nhi->ops->suspend_noirq) { 915 ret = nhi->ops->suspend_noirq(tb->nhi, wakeup); 916 if (ret) 917 return ret; 918 } 919 920 return 0; 921 } 922 923 static int nhi_suspend_noirq(struct device *dev) 924 { 925 return __nhi_suspend_noirq(dev, device_may_wakeup(dev)); 926 } 927 928 static int nhi_freeze_noirq(struct device *dev) 929 { 930 struct pci_dev *pdev = to_pci_dev(dev); 931 struct tb *tb = pci_get_drvdata(pdev); 932 933 return tb_domain_freeze_noirq(tb); 934 } 935 936 static int nhi_thaw_noirq(struct device *dev) 937 { 938 struct pci_dev *pdev = to_pci_dev(dev); 939 struct tb *tb = pci_get_drvdata(pdev); 940 941 return tb_domain_thaw_noirq(tb); 942 } 943 944 static bool nhi_wake_supported(struct pci_dev *pdev) 945 { 946 u8 val; 947 948 /* 949 * If power rails are sustainable for wakeup from S4 this 950 * property is set by the BIOS. 951 */ 952 if (device_property_read_u8(&pdev->dev, "WAKE_SUPPORTED", &val)) 953 return !!val; 954 955 return true; 956 } 957 958 static int nhi_poweroff_noirq(struct device *dev) 959 { 960 struct pci_dev *pdev = to_pci_dev(dev); 961 bool wakeup; 962 963 wakeup = device_may_wakeup(dev) && nhi_wake_supported(pdev); 964 return __nhi_suspend_noirq(dev, wakeup); 965 } 966 967 static void nhi_enable_int_throttling(struct tb_nhi *nhi) 968 { 969 /* Throttling is specified in 256ns increments */ 970 u32 throttle = DIV_ROUND_UP(128 * NSEC_PER_USEC, 256); 971 unsigned int i; 972 973 /* 974 * Configure interrupt throttling for all vectors even if we 975 * only use few. 976 */ 977 for (i = 0; i < MSIX_MAX_VECS; i++) { 978 u32 reg = REG_INT_THROTTLING_RATE + i * 4; 979 iowrite32(throttle, nhi->iobase + reg); 980 } 981 } 982 983 static int nhi_resume_noirq(struct device *dev) 984 { 985 struct pci_dev *pdev = to_pci_dev(dev); 986 struct tb *tb = pci_get_drvdata(pdev); 987 struct tb_nhi *nhi = tb->nhi; 988 int ret; 989 990 /* 991 * Check that the device is still there. It may be that the user 992 * unplugged last device which causes the host controller to go 993 * away on PCs. 994 */ 995 if (!pci_device_is_present(pdev)) { 996 nhi->going_away = true; 997 } else { 998 if (nhi->ops && nhi->ops->resume_noirq) { 999 ret = nhi->ops->resume_noirq(nhi); 1000 if (ret) 1001 return ret; 1002 } 1003 nhi_enable_int_throttling(tb->nhi); 1004 } 1005 1006 return tb_domain_resume_noirq(tb); 1007 } 1008 1009 static int nhi_suspend(struct device *dev) 1010 { 1011 struct pci_dev *pdev = to_pci_dev(dev); 1012 struct tb *tb = pci_get_drvdata(pdev); 1013 1014 return tb_domain_suspend(tb); 1015 } 1016 1017 static void nhi_complete(struct device *dev) 1018 { 1019 struct pci_dev *pdev = to_pci_dev(dev); 1020 struct tb *tb = pci_get_drvdata(pdev); 1021 1022 /* 1023 * If we were runtime suspended when system suspend started, 1024 * schedule runtime resume now. It should bring the domain back 1025 * to functional state. 1026 */ 1027 if (pm_runtime_suspended(&pdev->dev)) 1028 pm_runtime_resume(&pdev->dev); 1029 else 1030 tb_domain_complete(tb); 1031 } 1032 1033 static int nhi_runtime_suspend(struct device *dev) 1034 { 1035 struct pci_dev *pdev = to_pci_dev(dev); 1036 struct tb *tb = pci_get_drvdata(pdev); 1037 struct tb_nhi *nhi = tb->nhi; 1038 int ret; 1039 1040 ret = tb_domain_runtime_suspend(tb); 1041 if (ret) 1042 return ret; 1043 1044 if (nhi->ops && nhi->ops->runtime_suspend) { 1045 ret = nhi->ops->runtime_suspend(tb->nhi); 1046 if (ret) 1047 return ret; 1048 } 1049 return 0; 1050 } 1051 1052 static int nhi_runtime_resume(struct device *dev) 1053 { 1054 struct pci_dev *pdev = to_pci_dev(dev); 1055 struct tb *tb = pci_get_drvdata(pdev); 1056 struct tb_nhi *nhi = tb->nhi; 1057 int ret; 1058 1059 if (nhi->ops && nhi->ops->runtime_resume) { 1060 ret = nhi->ops->runtime_resume(nhi); 1061 if (ret) 1062 return ret; 1063 } 1064 1065 nhi_enable_int_throttling(nhi); 1066 return tb_domain_runtime_resume(tb); 1067 } 1068 1069 static void nhi_shutdown(struct tb_nhi *nhi) 1070 { 1071 int i; 1072 1073 dev_dbg(&nhi->pdev->dev, "shutdown\n"); 1074 1075 for (i = 0; i < nhi->hop_count; i++) { 1076 if (nhi->tx_rings[i]) 1077 dev_WARN(&nhi->pdev->dev, 1078 "TX ring %d is still active\n", i); 1079 if (nhi->rx_rings[i]) 1080 dev_WARN(&nhi->pdev->dev, 1081 "RX ring %d is still active\n", i); 1082 } 1083 nhi_disable_interrupts(nhi); 1084 /* 1085 * We have to release the irq before calling flush_work. Otherwise an 1086 * already executing IRQ handler could call schedule_work again. 1087 */ 1088 if (!nhi->pdev->msix_enabled) { 1089 devm_free_irq(&nhi->pdev->dev, nhi->pdev->irq, nhi); 1090 flush_work(&nhi->interrupt_work); 1091 } 1092 ida_destroy(&nhi->msix_ida); 1093 1094 if (nhi->ops && nhi->ops->shutdown) 1095 nhi->ops->shutdown(nhi); 1096 } 1097 1098 static void nhi_check_quirks(struct tb_nhi *nhi) 1099 { 1100 /* 1101 * Intel hardware supports auto clear of the interrupt status 1102 * reqister right after interrupt is being issued. 1103 */ 1104 if (nhi->pdev->vendor == PCI_VENDOR_ID_INTEL) 1105 nhi->quirks |= QUIRK_AUTO_CLEAR_INT; 1106 } 1107 1108 static int nhi_check_iommu_pdev(struct pci_dev *pdev, void *data) 1109 { 1110 if (!pdev->external_facing || 1111 !device_iommu_capable(&pdev->dev, IOMMU_CAP_PRE_BOOT_PROTECTION)) 1112 return 0; 1113 *(bool *)data = true; 1114 return 1; /* Stop walking */ 1115 } 1116 1117 static void nhi_check_iommu(struct tb_nhi *nhi) 1118 { 1119 struct pci_bus *bus = nhi->pdev->bus; 1120 bool port_ok = false; 1121 1122 /* 1123 * Ideally what we'd do here is grab every PCI device that 1124 * represents a tunnelling adapter for this NHI and check their 1125 * status directly, but unfortunately USB4 seems to make it 1126 * obnoxiously difficult to reliably make any correlation. 1127 * 1128 * So for now we'll have to bodge it... Hoping that the system 1129 * is at least sane enough that an adapter is in the same PCI 1130 * segment as its NHI, if we can find *something* on that segment 1131 * which meets the requirements for Kernel DMA Protection, we'll 1132 * take that to imply that firmware is aware and has (hopefully) 1133 * done the right thing in general. We need to know that the PCI 1134 * layer has seen the ExternalFacingPort property which will then 1135 * inform the IOMMU layer to enforce the complete "untrusted DMA" 1136 * flow, but also that the IOMMU driver itself can be trusted not 1137 * to have been subverted by a pre-boot DMA attack. 1138 */ 1139 while (bus->parent) 1140 bus = bus->parent; 1141 1142 pci_walk_bus(bus, nhi_check_iommu_pdev, &port_ok); 1143 1144 nhi->iommu_dma_protection = port_ok; 1145 dev_dbg(&nhi->pdev->dev, "IOMMU DMA protection is %s\n", 1146 str_enabled_disabled(port_ok)); 1147 } 1148 1149 static int nhi_init_msi(struct tb_nhi *nhi) 1150 { 1151 struct pci_dev *pdev = nhi->pdev; 1152 int res, irq, nvec; 1153 1154 /* In case someone left them on. */ 1155 nhi_disable_interrupts(nhi); 1156 1157 nhi_enable_int_throttling(nhi); 1158 1159 ida_init(&nhi->msix_ida); 1160 1161 /* 1162 * The NHI has 16 MSI-X vectors or a single MSI. We first try to 1163 * get all MSI-X vectors and if we succeed, each ring will have 1164 * one MSI-X. If for some reason that does not work out, we 1165 * fallback to a single MSI. 1166 */ 1167 nvec = pci_alloc_irq_vectors(pdev, MSIX_MIN_VECS, MSIX_MAX_VECS, 1168 PCI_IRQ_MSIX); 1169 if (nvec < 0) { 1170 nvec = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); 1171 if (nvec < 0) 1172 return nvec; 1173 1174 INIT_WORK(&nhi->interrupt_work, nhi_interrupt_work); 1175 1176 irq = pci_irq_vector(nhi->pdev, 0); 1177 if (irq < 0) 1178 return irq; 1179 1180 res = devm_request_irq(&pdev->dev, irq, nhi_msi, 1181 IRQF_NO_SUSPEND, "thunderbolt", nhi); 1182 if (res) { 1183 dev_err(&pdev->dev, "request_irq failed, aborting\n"); 1184 return res; 1185 } 1186 } 1187 1188 return 0; 1189 } 1190 1191 static bool nhi_imr_valid(struct pci_dev *pdev) 1192 { 1193 u8 val; 1194 1195 if (!device_property_read_u8(&pdev->dev, "IMR_VALID", &val)) 1196 return !!val; 1197 1198 return true; 1199 } 1200 1201 static struct tb *nhi_select_cm(struct tb_nhi *nhi) 1202 { 1203 struct tb *tb; 1204 1205 /* 1206 * USB4 case is simple. If we got control of any of the 1207 * capabilities, we use software CM. 1208 */ 1209 if (tb_acpi_is_native()) 1210 return tb_probe(nhi); 1211 1212 /* 1213 * Either firmware based CM is running (we did not get control 1214 * from the firmware) or this is pre-USB4 PC so try first 1215 * firmware CM and then fallback to software CM. 1216 */ 1217 tb = icm_probe(nhi); 1218 if (!tb) 1219 tb = tb_probe(nhi); 1220 1221 return tb; 1222 } 1223 1224 static int nhi_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1225 { 1226 struct tb_nhi *nhi; 1227 struct tb *tb; 1228 int res; 1229 1230 if (!nhi_imr_valid(pdev)) { 1231 dev_warn(&pdev->dev, "firmware image not valid, aborting\n"); 1232 return -ENODEV; 1233 } 1234 1235 res = pcim_enable_device(pdev); 1236 if (res) { 1237 dev_err(&pdev->dev, "cannot enable PCI device, aborting\n"); 1238 return res; 1239 } 1240 1241 res = pcim_iomap_regions(pdev, 1 << 0, "thunderbolt"); 1242 if (res) { 1243 dev_err(&pdev->dev, "cannot obtain PCI resources, aborting\n"); 1244 return res; 1245 } 1246 1247 nhi = devm_kzalloc(&pdev->dev, sizeof(*nhi), GFP_KERNEL); 1248 if (!nhi) 1249 return -ENOMEM; 1250 1251 nhi->pdev = pdev; 1252 nhi->ops = (const struct tb_nhi_ops *)id->driver_data; 1253 /* cannot fail - table is allocated in pcim_iomap_regions */ 1254 nhi->iobase = pcim_iomap_table(pdev)[0]; 1255 nhi->hop_count = ioread32(nhi->iobase + REG_HOP_COUNT) & 0x3ff; 1256 dev_dbg(&pdev->dev, "total paths: %d\n", nhi->hop_count); 1257 1258 nhi->tx_rings = devm_kcalloc(&pdev->dev, nhi->hop_count, 1259 sizeof(*nhi->tx_rings), GFP_KERNEL); 1260 nhi->rx_rings = devm_kcalloc(&pdev->dev, nhi->hop_count, 1261 sizeof(*nhi->rx_rings), GFP_KERNEL); 1262 if (!nhi->tx_rings || !nhi->rx_rings) 1263 return -ENOMEM; 1264 1265 nhi_check_quirks(nhi); 1266 nhi_check_iommu(nhi); 1267 1268 res = nhi_init_msi(nhi); 1269 if (res) { 1270 dev_err(&pdev->dev, "cannot enable MSI, aborting\n"); 1271 return res; 1272 } 1273 1274 spin_lock_init(&nhi->lock); 1275 1276 res = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1277 if (res) { 1278 dev_err(&pdev->dev, "failed to set DMA mask\n"); 1279 return res; 1280 } 1281 1282 pci_set_master(pdev); 1283 1284 if (nhi->ops && nhi->ops->init) { 1285 res = nhi->ops->init(nhi); 1286 if (res) 1287 return res; 1288 } 1289 1290 tb = nhi_select_cm(nhi); 1291 if (!tb) { 1292 dev_err(&nhi->pdev->dev, 1293 "failed to determine connection manager, aborting\n"); 1294 return -ENODEV; 1295 } 1296 1297 dev_dbg(&nhi->pdev->dev, "NHI initialized, starting thunderbolt\n"); 1298 1299 res = tb_domain_add(tb); 1300 if (res) { 1301 /* 1302 * At this point the RX/TX rings might already have been 1303 * activated. Do a proper shutdown. 1304 */ 1305 tb_domain_put(tb); 1306 nhi_shutdown(nhi); 1307 return res; 1308 } 1309 pci_set_drvdata(pdev, tb); 1310 1311 device_wakeup_enable(&pdev->dev); 1312 1313 pm_runtime_allow(&pdev->dev); 1314 pm_runtime_set_autosuspend_delay(&pdev->dev, TB_AUTOSUSPEND_DELAY); 1315 pm_runtime_use_autosuspend(&pdev->dev); 1316 pm_runtime_put_autosuspend(&pdev->dev); 1317 1318 return 0; 1319 } 1320 1321 static void nhi_remove(struct pci_dev *pdev) 1322 { 1323 struct tb *tb = pci_get_drvdata(pdev); 1324 struct tb_nhi *nhi = tb->nhi; 1325 1326 pm_runtime_get_sync(&pdev->dev); 1327 pm_runtime_dont_use_autosuspend(&pdev->dev); 1328 pm_runtime_forbid(&pdev->dev); 1329 1330 tb_domain_remove(tb); 1331 nhi_shutdown(nhi); 1332 } 1333 1334 /* 1335 * The tunneled pci bridges are siblings of us. Use resume_noirq to reenable 1336 * the tunnels asap. A corresponding pci quirk blocks the downstream bridges 1337 * resume_noirq until we are done. 1338 */ 1339 static const struct dev_pm_ops nhi_pm_ops = { 1340 .suspend_noirq = nhi_suspend_noirq, 1341 .resume_noirq = nhi_resume_noirq, 1342 .freeze_noirq = nhi_freeze_noirq, /* 1343 * we just disable hotplug, the 1344 * pci-tunnels stay alive. 1345 */ 1346 .thaw_noirq = nhi_thaw_noirq, 1347 .restore_noirq = nhi_resume_noirq, 1348 .suspend = nhi_suspend, 1349 .poweroff_noirq = nhi_poweroff_noirq, 1350 .poweroff = nhi_suspend, 1351 .complete = nhi_complete, 1352 .runtime_suspend = nhi_runtime_suspend, 1353 .runtime_resume = nhi_runtime_resume, 1354 }; 1355 1356 static struct pci_device_id nhi_ids[] = { 1357 /* 1358 * We have to specify class, the TB bridges use the same device and 1359 * vendor (sub)id on gen 1 and gen 2 controllers. 1360 */ 1361 { 1362 .class = PCI_CLASS_SYSTEM_OTHER << 8, .class_mask = ~0, 1363 .vendor = PCI_VENDOR_ID_INTEL, 1364 .device = PCI_DEVICE_ID_INTEL_LIGHT_RIDGE, 1365 .subvendor = 0x2222, .subdevice = 0x1111, 1366 }, 1367 { 1368 .class = PCI_CLASS_SYSTEM_OTHER << 8, .class_mask = ~0, 1369 .vendor = PCI_VENDOR_ID_INTEL, 1370 .device = PCI_DEVICE_ID_INTEL_CACTUS_RIDGE_4C, 1371 .subvendor = 0x2222, .subdevice = 0x1111, 1372 }, 1373 { 1374 .class = PCI_CLASS_SYSTEM_OTHER << 8, .class_mask = ~0, 1375 .vendor = PCI_VENDOR_ID_INTEL, 1376 .device = PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI, 1377 .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID, 1378 }, 1379 { 1380 .class = PCI_CLASS_SYSTEM_OTHER << 8, .class_mask = ~0, 1381 .vendor = PCI_VENDOR_ID_INTEL, 1382 .device = PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI, 1383 .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID, 1384 }, 1385 1386 /* Thunderbolt 3 */ 1387 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_NHI) }, 1388 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_NHI) }, 1389 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_USBONLY_NHI) }, 1390 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_NHI) }, 1391 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_USBONLY_NHI) }, 1392 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_NHI) }, 1393 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_NHI) }, 1394 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_USBONLY_NHI) }, 1395 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_NHI) }, 1396 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_NHI) }, 1397 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ICL_NHI0), 1398 .driver_data = (kernel_ulong_t)&icl_nhi_ops }, 1399 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ICL_NHI1), 1400 .driver_data = (kernel_ulong_t)&icl_nhi_ops }, 1401 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TGL_NHI0), 1402 .driver_data = (kernel_ulong_t)&icl_nhi_ops }, 1403 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TGL_NHI1), 1404 .driver_data = (kernel_ulong_t)&icl_nhi_ops }, 1405 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TGL_H_NHI0), 1406 .driver_data = (kernel_ulong_t)&icl_nhi_ops }, 1407 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TGL_H_NHI1), 1408 .driver_data = (kernel_ulong_t)&icl_nhi_ops }, 1409 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADL_NHI0), 1410 .driver_data = (kernel_ulong_t)&icl_nhi_ops }, 1411 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADL_NHI1), 1412 .driver_data = (kernel_ulong_t)&icl_nhi_ops }, 1413 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_RPL_NHI0), 1414 .driver_data = (kernel_ulong_t)&icl_nhi_ops }, 1415 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_RPL_NHI1), 1416 .driver_data = (kernel_ulong_t)&icl_nhi_ops }, 1417 1418 /* Any USB4 compliant host */ 1419 { PCI_DEVICE_CLASS(PCI_CLASS_SERIAL_USB_USB4, ~0) }, 1420 1421 { 0,} 1422 }; 1423 1424 MODULE_DEVICE_TABLE(pci, nhi_ids); 1425 MODULE_LICENSE("GPL"); 1426 1427 static struct pci_driver nhi_driver = { 1428 .name = "thunderbolt", 1429 .id_table = nhi_ids, 1430 .probe = nhi_probe, 1431 .remove = nhi_remove, 1432 .shutdown = nhi_remove, 1433 .driver.pm = &nhi_pm_ops, 1434 }; 1435 1436 static int __init nhi_init(void) 1437 { 1438 int ret; 1439 1440 ret = tb_domain_init(); 1441 if (ret) 1442 return ret; 1443 ret = pci_register_driver(&nhi_driver); 1444 if (ret) 1445 tb_domain_exit(); 1446 return ret; 1447 } 1448 1449 static void __exit nhi_unload(void) 1450 { 1451 pci_unregister_driver(&nhi_driver); 1452 tb_domain_exit(); 1453 } 1454 1455 rootfs_initcall(nhi_init); 1456 module_exit(nhi_unload); 1457