1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Thunderbolt driver - bus logic (NHI independent) 4 * 5 * Copyright (c) 2014 Andreas Noever <andreas.noever@gmail.com> 6 * Copyright (C) 2019, Intel Corporation 7 */ 8 9 #include <linux/slab.h> 10 #include <linux/errno.h> 11 #include <linux/delay.h> 12 13 #include "tb.h" 14 #include "tb_regs.h" 15 #include "tunnel.h" 16 17 /** 18 * struct tb_cm - Simple Thunderbolt connection manager 19 * @tunnel_list: List of active tunnels 20 * @dp_resources: List of available DP resources for DP tunneling 21 * @hotplug_active: tb_handle_hotplug will stop progressing plug 22 * events and exit if this is not set (it needs to 23 * acquire the lock one more time). Used to drain wq 24 * after cfg has been paused. 25 */ 26 struct tb_cm { 27 struct list_head tunnel_list; 28 struct list_head dp_resources; 29 bool hotplug_active; 30 }; 31 32 struct tb_hotplug_event { 33 struct work_struct work; 34 struct tb *tb; 35 u64 route; 36 u8 port; 37 bool unplug; 38 }; 39 40 static void tb_handle_hotplug(struct work_struct *work); 41 42 static void tb_queue_hotplug(struct tb *tb, u64 route, u8 port, bool unplug) 43 { 44 struct tb_hotplug_event *ev; 45 46 ev = kmalloc(sizeof(*ev), GFP_KERNEL); 47 if (!ev) 48 return; 49 50 ev->tb = tb; 51 ev->route = route; 52 ev->port = port; 53 ev->unplug = unplug; 54 INIT_WORK(&ev->work, tb_handle_hotplug); 55 queue_work(tb->wq, &ev->work); 56 } 57 58 /* enumeration & hot plug handling */ 59 60 static void tb_add_dp_resources(struct tb_switch *sw) 61 { 62 struct tb_cm *tcm = tb_priv(sw->tb); 63 struct tb_port *port; 64 65 tb_switch_for_each_port(sw, port) { 66 if (!tb_port_is_dpin(port)) 67 continue; 68 69 if (!tb_switch_query_dp_resource(sw, port)) 70 continue; 71 72 list_add_tail(&port->list, &tcm->dp_resources); 73 tb_port_dbg(port, "DP IN resource available\n"); 74 } 75 } 76 77 static void tb_remove_dp_resources(struct tb_switch *sw) 78 { 79 struct tb_cm *tcm = tb_priv(sw->tb); 80 struct tb_port *port, *tmp; 81 82 /* Clear children resources first */ 83 tb_switch_for_each_port(sw, port) { 84 if (tb_port_has_remote(port)) 85 tb_remove_dp_resources(port->remote->sw); 86 } 87 88 list_for_each_entry_safe(port, tmp, &tcm->dp_resources, list) { 89 if (port->sw == sw) { 90 tb_port_dbg(port, "DP OUT resource unavailable\n"); 91 list_del_init(&port->list); 92 } 93 } 94 } 95 96 static void tb_discover_tunnels(struct tb_switch *sw) 97 { 98 struct tb *tb = sw->tb; 99 struct tb_cm *tcm = tb_priv(tb); 100 struct tb_port *port; 101 102 tb_switch_for_each_port(sw, port) { 103 struct tb_tunnel *tunnel = NULL; 104 105 switch (port->config.type) { 106 case TB_TYPE_DP_HDMI_IN: 107 tunnel = tb_tunnel_discover_dp(tb, port); 108 break; 109 110 case TB_TYPE_PCIE_DOWN: 111 tunnel = tb_tunnel_discover_pci(tb, port); 112 break; 113 114 case TB_TYPE_USB3_DOWN: 115 tunnel = tb_tunnel_discover_usb3(tb, port); 116 break; 117 118 default: 119 break; 120 } 121 122 if (!tunnel) 123 continue; 124 125 if (tb_tunnel_is_pci(tunnel)) { 126 struct tb_switch *parent = tunnel->dst_port->sw; 127 128 while (parent != tunnel->src_port->sw) { 129 parent->boot = true; 130 parent = tb_switch_parent(parent); 131 } 132 } 133 134 list_add_tail(&tunnel->list, &tcm->tunnel_list); 135 } 136 137 tb_switch_for_each_port(sw, port) { 138 if (tb_port_has_remote(port)) 139 tb_discover_tunnels(port->remote->sw); 140 } 141 } 142 143 static void tb_scan_xdomain(struct tb_port *port) 144 { 145 struct tb_switch *sw = port->sw; 146 struct tb *tb = sw->tb; 147 struct tb_xdomain *xd; 148 u64 route; 149 150 route = tb_downstream_route(port); 151 xd = tb_xdomain_find_by_route(tb, route); 152 if (xd) { 153 tb_xdomain_put(xd); 154 return; 155 } 156 157 xd = tb_xdomain_alloc(tb, &sw->dev, route, tb->root_switch->uuid, 158 NULL); 159 if (xd) { 160 tb_port_at(route, sw)->xdomain = xd; 161 tb_xdomain_add(xd); 162 } 163 } 164 165 static int tb_enable_tmu(struct tb_switch *sw) 166 { 167 int ret; 168 169 /* If it is already enabled in correct mode, don't touch it */ 170 if (tb_switch_tmu_is_enabled(sw)) 171 return 0; 172 173 ret = tb_switch_tmu_disable(sw); 174 if (ret) 175 return ret; 176 177 ret = tb_switch_tmu_post_time(sw); 178 if (ret) 179 return ret; 180 181 return tb_switch_tmu_enable(sw); 182 } 183 184 /** 185 * tb_find_unused_port() - return the first inactive port on @sw 186 * @sw: Switch to find the port on 187 * @type: Port type to look for 188 */ 189 static struct tb_port *tb_find_unused_port(struct tb_switch *sw, 190 enum tb_port_type type) 191 { 192 struct tb_port *port; 193 194 tb_switch_for_each_port(sw, port) { 195 if (tb_is_upstream_port(port)) 196 continue; 197 if (port->config.type != type) 198 continue; 199 if (!port->cap_adap) 200 continue; 201 if (tb_port_is_enabled(port)) 202 continue; 203 return port; 204 } 205 return NULL; 206 } 207 208 static struct tb_port *tb_find_usb3_down(struct tb_switch *sw, 209 const struct tb_port *port) 210 { 211 struct tb_port *down; 212 213 down = usb4_switch_map_usb3_down(sw, port); 214 if (down && !tb_usb3_port_is_enabled(down)) 215 return down; 216 return NULL; 217 } 218 219 static struct tb_tunnel *tb_find_tunnel(struct tb *tb, enum tb_tunnel_type type, 220 struct tb_port *src_port, 221 struct tb_port *dst_port) 222 { 223 struct tb_cm *tcm = tb_priv(tb); 224 struct tb_tunnel *tunnel; 225 226 list_for_each_entry(tunnel, &tcm->tunnel_list, list) { 227 if (tunnel->type == type && 228 ((src_port && src_port == tunnel->src_port) || 229 (dst_port && dst_port == tunnel->dst_port))) { 230 return tunnel; 231 } 232 } 233 234 return NULL; 235 } 236 237 static struct tb_tunnel *tb_find_first_usb3_tunnel(struct tb *tb, 238 struct tb_port *src_port, 239 struct tb_port *dst_port) 240 { 241 struct tb_port *port, *usb3_down; 242 struct tb_switch *sw; 243 244 /* Pick the router that is deepest in the topology */ 245 if (dst_port->sw->config.depth > src_port->sw->config.depth) 246 sw = dst_port->sw; 247 else 248 sw = src_port->sw; 249 250 /* Can't be the host router */ 251 if (sw == tb->root_switch) 252 return NULL; 253 254 /* Find the downstream USB4 port that leads to this router */ 255 port = tb_port_at(tb_route(sw), tb->root_switch); 256 /* Find the corresponding host router USB3 downstream port */ 257 usb3_down = usb4_switch_map_usb3_down(tb->root_switch, port); 258 if (!usb3_down) 259 return NULL; 260 261 return tb_find_tunnel(tb, TB_TUNNEL_USB3, usb3_down, NULL); 262 } 263 264 static int tb_available_bandwidth(struct tb *tb, struct tb_port *src_port, 265 struct tb_port *dst_port, int *available_up, int *available_down) 266 { 267 int usb3_consumed_up, usb3_consumed_down, ret; 268 struct tb_cm *tcm = tb_priv(tb); 269 struct tb_tunnel *tunnel; 270 struct tb_port *port; 271 272 tb_port_dbg(dst_port, "calculating available bandwidth\n"); 273 274 tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port); 275 if (tunnel) { 276 ret = tb_tunnel_consumed_bandwidth(tunnel, &usb3_consumed_up, 277 &usb3_consumed_down); 278 if (ret) 279 return ret; 280 } else { 281 usb3_consumed_up = 0; 282 usb3_consumed_down = 0; 283 } 284 285 *available_up = *available_down = 40000; 286 287 /* Find the minimum available bandwidth over all links */ 288 tb_for_each_port_on_path(src_port, dst_port, port) { 289 int link_speed, link_width, up_bw, down_bw; 290 291 if (!tb_port_is_null(port)) 292 continue; 293 294 if (tb_is_upstream_port(port)) { 295 link_speed = port->sw->link_speed; 296 } else { 297 link_speed = tb_port_get_link_speed(port); 298 if (link_speed < 0) 299 return link_speed; 300 } 301 302 link_width = port->bonded ? 2 : 1; 303 304 up_bw = link_speed * link_width * 1000; /* Mb/s */ 305 /* Leave 10% guard band */ 306 up_bw -= up_bw / 10; 307 down_bw = up_bw; 308 309 tb_port_dbg(port, "link total bandwidth %d Mb/s\n", up_bw); 310 311 /* 312 * Find all DP tunnels that cross the port and reduce 313 * their consumed bandwidth from the available. 314 */ 315 list_for_each_entry(tunnel, &tcm->tunnel_list, list) { 316 int dp_consumed_up, dp_consumed_down; 317 318 if (!tb_tunnel_is_dp(tunnel)) 319 continue; 320 321 if (!tb_tunnel_port_on_path(tunnel, port)) 322 continue; 323 324 ret = tb_tunnel_consumed_bandwidth(tunnel, 325 &dp_consumed_up, 326 &dp_consumed_down); 327 if (ret) 328 return ret; 329 330 up_bw -= dp_consumed_up; 331 down_bw -= dp_consumed_down; 332 } 333 334 /* 335 * If USB3 is tunneled from the host router down to the 336 * branch leading to port we need to take USB3 consumed 337 * bandwidth into account regardless whether it actually 338 * crosses the port. 339 */ 340 up_bw -= usb3_consumed_up; 341 down_bw -= usb3_consumed_down; 342 343 if (up_bw < *available_up) 344 *available_up = up_bw; 345 if (down_bw < *available_down) 346 *available_down = down_bw; 347 } 348 349 if (*available_up < 0) 350 *available_up = 0; 351 if (*available_down < 0) 352 *available_down = 0; 353 354 return 0; 355 } 356 357 static int tb_release_unused_usb3_bandwidth(struct tb *tb, 358 struct tb_port *src_port, 359 struct tb_port *dst_port) 360 { 361 struct tb_tunnel *tunnel; 362 363 tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port); 364 return tunnel ? tb_tunnel_release_unused_bandwidth(tunnel) : 0; 365 } 366 367 static void tb_reclaim_usb3_bandwidth(struct tb *tb, struct tb_port *src_port, 368 struct tb_port *dst_port) 369 { 370 int ret, available_up, available_down; 371 struct tb_tunnel *tunnel; 372 373 tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port); 374 if (!tunnel) 375 return; 376 377 tb_dbg(tb, "reclaiming unused bandwidth for USB3\n"); 378 379 /* 380 * Calculate available bandwidth for the first hop USB3 tunnel. 381 * That determines the whole USB3 bandwidth for this branch. 382 */ 383 ret = tb_available_bandwidth(tb, tunnel->src_port, tunnel->dst_port, 384 &available_up, &available_down); 385 if (ret) { 386 tb_warn(tb, "failed to calculate available bandwidth\n"); 387 return; 388 } 389 390 tb_dbg(tb, "available bandwidth for USB3 %d/%d Mb/s\n", 391 available_up, available_down); 392 393 tb_tunnel_reclaim_available_bandwidth(tunnel, &available_up, &available_down); 394 } 395 396 static int tb_tunnel_usb3(struct tb *tb, struct tb_switch *sw) 397 { 398 struct tb_switch *parent = tb_switch_parent(sw); 399 int ret, available_up, available_down; 400 struct tb_port *up, *down, *port; 401 struct tb_cm *tcm = tb_priv(tb); 402 struct tb_tunnel *tunnel; 403 404 up = tb_switch_find_port(sw, TB_TYPE_USB3_UP); 405 if (!up) 406 return 0; 407 408 if (!sw->link_usb4) 409 return 0; 410 411 /* 412 * Look up available down port. Since we are chaining it should 413 * be found right above this switch. 414 */ 415 port = tb_port_at(tb_route(sw), parent); 416 down = tb_find_usb3_down(parent, port); 417 if (!down) 418 return 0; 419 420 if (tb_route(parent)) { 421 struct tb_port *parent_up; 422 /* 423 * Check first that the parent switch has its upstream USB3 424 * port enabled. Otherwise the chain is not complete and 425 * there is no point setting up a new tunnel. 426 */ 427 parent_up = tb_switch_find_port(parent, TB_TYPE_USB3_UP); 428 if (!parent_up || !tb_port_is_enabled(parent_up)) 429 return 0; 430 431 /* Make all unused bandwidth available for the new tunnel */ 432 ret = tb_release_unused_usb3_bandwidth(tb, down, up); 433 if (ret) 434 return ret; 435 } 436 437 ret = tb_available_bandwidth(tb, down, up, &available_up, 438 &available_down); 439 if (ret) 440 goto err_reclaim; 441 442 tb_port_dbg(up, "available bandwidth for new USB3 tunnel %d/%d Mb/s\n", 443 available_up, available_down); 444 445 tunnel = tb_tunnel_alloc_usb3(tb, up, down, available_up, 446 available_down); 447 if (!tunnel) { 448 ret = -ENOMEM; 449 goto err_reclaim; 450 } 451 452 if (tb_tunnel_activate(tunnel)) { 453 tb_port_info(up, 454 "USB3 tunnel activation failed, aborting\n"); 455 ret = -EIO; 456 goto err_free; 457 } 458 459 list_add_tail(&tunnel->list, &tcm->tunnel_list); 460 if (tb_route(parent)) 461 tb_reclaim_usb3_bandwidth(tb, down, up); 462 463 return 0; 464 465 err_free: 466 tb_tunnel_free(tunnel); 467 err_reclaim: 468 if (tb_route(parent)) 469 tb_reclaim_usb3_bandwidth(tb, down, up); 470 471 return ret; 472 } 473 474 static int tb_create_usb3_tunnels(struct tb_switch *sw) 475 { 476 struct tb_port *port; 477 int ret; 478 479 if (tb_route(sw)) { 480 ret = tb_tunnel_usb3(sw->tb, sw); 481 if (ret) 482 return ret; 483 } 484 485 tb_switch_for_each_port(sw, port) { 486 if (!tb_port_has_remote(port)) 487 continue; 488 ret = tb_create_usb3_tunnels(port->remote->sw); 489 if (ret) 490 return ret; 491 } 492 493 return 0; 494 } 495 496 static void tb_scan_port(struct tb_port *port); 497 498 /** 499 * tb_scan_switch() - scan for and initialize downstream switches 500 */ 501 static void tb_scan_switch(struct tb_switch *sw) 502 { 503 struct tb_port *port; 504 505 tb_switch_for_each_port(sw, port) 506 tb_scan_port(port); 507 } 508 509 /** 510 * tb_scan_port() - check for and initialize switches below port 511 */ 512 static void tb_scan_port(struct tb_port *port) 513 { 514 struct tb_cm *tcm = tb_priv(port->sw->tb); 515 struct tb_port *upstream_port; 516 struct tb_switch *sw; 517 518 if (tb_is_upstream_port(port)) 519 return; 520 521 if (tb_port_is_dpout(port) && tb_dp_port_hpd_is_active(port) == 1 && 522 !tb_dp_port_is_enabled(port)) { 523 tb_port_dbg(port, "DP adapter HPD set, queuing hotplug\n"); 524 tb_queue_hotplug(port->sw->tb, tb_route(port->sw), port->port, 525 false); 526 return; 527 } 528 529 if (port->config.type != TB_TYPE_PORT) 530 return; 531 if (port->dual_link_port && port->link_nr) 532 return; /* 533 * Downstream switch is reachable through two ports. 534 * Only scan on the primary port (link_nr == 0). 535 */ 536 if (tb_wait_for_port(port, false) <= 0) 537 return; 538 if (port->remote) { 539 tb_port_dbg(port, "port already has a remote\n"); 540 return; 541 } 542 543 tb_retimer_scan(port); 544 545 sw = tb_switch_alloc(port->sw->tb, &port->sw->dev, 546 tb_downstream_route(port)); 547 if (IS_ERR(sw)) { 548 /* 549 * If there is an error accessing the connected switch 550 * it may be connected to another domain. Also we allow 551 * the other domain to be connected to a max depth switch. 552 */ 553 if (PTR_ERR(sw) == -EIO || PTR_ERR(sw) == -EADDRNOTAVAIL) 554 tb_scan_xdomain(port); 555 return; 556 } 557 558 if (tb_switch_configure(sw)) { 559 tb_switch_put(sw); 560 return; 561 } 562 563 /* 564 * If there was previously another domain connected remove it 565 * first. 566 */ 567 if (port->xdomain) { 568 tb_xdomain_remove(port->xdomain); 569 port->xdomain = NULL; 570 } 571 572 /* 573 * Do not send uevents until we have discovered all existing 574 * tunnels and know which switches were authorized already by 575 * the boot firmware. 576 */ 577 if (!tcm->hotplug_active) 578 dev_set_uevent_suppress(&sw->dev, true); 579 580 if (tb_switch_add(sw)) { 581 tb_switch_put(sw); 582 return; 583 } 584 585 /* Link the switches using both links if available */ 586 upstream_port = tb_upstream_port(sw); 587 port->remote = upstream_port; 588 upstream_port->remote = port; 589 if (port->dual_link_port && upstream_port->dual_link_port) { 590 port->dual_link_port->remote = upstream_port->dual_link_port; 591 upstream_port->dual_link_port->remote = port->dual_link_port; 592 } 593 594 /* Enable lane bonding if supported */ 595 tb_switch_lane_bonding_enable(sw); 596 597 if (tb_enable_tmu(sw)) 598 tb_sw_warn(sw, "failed to enable TMU\n"); 599 600 /* Scan upstream retimers */ 601 tb_retimer_scan(upstream_port); 602 603 /* 604 * Create USB 3.x tunnels only when the switch is plugged to the 605 * domain. This is because we scan the domain also during discovery 606 * and want to discover existing USB 3.x tunnels before we create 607 * any new. 608 */ 609 if (tcm->hotplug_active && tb_tunnel_usb3(sw->tb, sw)) 610 tb_sw_warn(sw, "USB3 tunnel creation failed\n"); 611 612 tb_add_dp_resources(sw); 613 tb_scan_switch(sw); 614 } 615 616 static void tb_deactivate_and_free_tunnel(struct tb_tunnel *tunnel) 617 { 618 struct tb_port *src_port, *dst_port; 619 struct tb *tb; 620 621 if (!tunnel) 622 return; 623 624 tb_tunnel_deactivate(tunnel); 625 list_del(&tunnel->list); 626 627 tb = tunnel->tb; 628 src_port = tunnel->src_port; 629 dst_port = tunnel->dst_port; 630 631 switch (tunnel->type) { 632 case TB_TUNNEL_DP: 633 /* 634 * In case of DP tunnel make sure the DP IN resource is 635 * deallocated properly. 636 */ 637 tb_switch_dealloc_dp_resource(src_port->sw, src_port); 638 fallthrough; 639 640 case TB_TUNNEL_USB3: 641 tb_reclaim_usb3_bandwidth(tb, src_port, dst_port); 642 break; 643 644 default: 645 /* 646 * PCIe and DMA tunnels do not consume guaranteed 647 * bandwidth. 648 */ 649 break; 650 } 651 652 tb_tunnel_free(tunnel); 653 } 654 655 /** 656 * tb_free_invalid_tunnels() - destroy tunnels of devices that have gone away 657 */ 658 static void tb_free_invalid_tunnels(struct tb *tb) 659 { 660 struct tb_cm *tcm = tb_priv(tb); 661 struct tb_tunnel *tunnel; 662 struct tb_tunnel *n; 663 664 list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) { 665 if (tb_tunnel_is_invalid(tunnel)) 666 tb_deactivate_and_free_tunnel(tunnel); 667 } 668 } 669 670 /** 671 * tb_free_unplugged_children() - traverse hierarchy and free unplugged switches 672 */ 673 static void tb_free_unplugged_children(struct tb_switch *sw) 674 { 675 struct tb_port *port; 676 677 tb_switch_for_each_port(sw, port) { 678 if (!tb_port_has_remote(port)) 679 continue; 680 681 if (port->remote->sw->is_unplugged) { 682 tb_retimer_remove_all(port); 683 tb_remove_dp_resources(port->remote->sw); 684 tb_switch_lane_bonding_disable(port->remote->sw); 685 tb_switch_remove(port->remote->sw); 686 port->remote = NULL; 687 if (port->dual_link_port) 688 port->dual_link_port->remote = NULL; 689 } else { 690 tb_free_unplugged_children(port->remote->sw); 691 } 692 } 693 } 694 695 static struct tb_port *tb_find_pcie_down(struct tb_switch *sw, 696 const struct tb_port *port) 697 { 698 struct tb_port *down = NULL; 699 700 /* 701 * To keep plugging devices consistently in the same PCIe 702 * hierarchy, do mapping here for switch downstream PCIe ports. 703 */ 704 if (tb_switch_is_usb4(sw)) { 705 down = usb4_switch_map_pcie_down(sw, port); 706 } else if (!tb_route(sw)) { 707 int phy_port = tb_phy_port_from_link(port->port); 708 int index; 709 710 /* 711 * Hard-coded Thunderbolt port to PCIe down port mapping 712 * per controller. 713 */ 714 if (tb_switch_is_cactus_ridge(sw) || 715 tb_switch_is_alpine_ridge(sw)) 716 index = !phy_port ? 6 : 7; 717 else if (tb_switch_is_falcon_ridge(sw)) 718 index = !phy_port ? 6 : 8; 719 else if (tb_switch_is_titan_ridge(sw)) 720 index = !phy_port ? 8 : 9; 721 else 722 goto out; 723 724 /* Validate the hard-coding */ 725 if (WARN_ON(index > sw->config.max_port_number)) 726 goto out; 727 728 down = &sw->ports[index]; 729 } 730 731 if (down) { 732 if (WARN_ON(!tb_port_is_pcie_down(down))) 733 goto out; 734 if (tb_pci_port_is_enabled(down)) 735 goto out; 736 737 return down; 738 } 739 740 out: 741 return tb_find_unused_port(sw, TB_TYPE_PCIE_DOWN); 742 } 743 744 static struct tb_port *tb_find_dp_out(struct tb *tb, struct tb_port *in) 745 { 746 struct tb_port *host_port, *port; 747 struct tb_cm *tcm = tb_priv(tb); 748 749 host_port = tb_route(in->sw) ? 750 tb_port_at(tb_route(in->sw), tb->root_switch) : NULL; 751 752 list_for_each_entry(port, &tcm->dp_resources, list) { 753 if (!tb_port_is_dpout(port)) 754 continue; 755 756 if (tb_port_is_enabled(port)) { 757 tb_port_dbg(port, "in use\n"); 758 continue; 759 } 760 761 tb_port_dbg(port, "DP OUT available\n"); 762 763 /* 764 * Keep the DP tunnel under the topology starting from 765 * the same host router downstream port. 766 */ 767 if (host_port && tb_route(port->sw)) { 768 struct tb_port *p; 769 770 p = tb_port_at(tb_route(port->sw), tb->root_switch); 771 if (p != host_port) 772 continue; 773 } 774 775 return port; 776 } 777 778 return NULL; 779 } 780 781 static void tb_tunnel_dp(struct tb *tb) 782 { 783 int available_up, available_down, ret; 784 struct tb_cm *tcm = tb_priv(tb); 785 struct tb_port *port, *in, *out; 786 struct tb_tunnel *tunnel; 787 788 /* 789 * Find pair of inactive DP IN and DP OUT adapters and then 790 * establish a DP tunnel between them. 791 */ 792 tb_dbg(tb, "looking for DP IN <-> DP OUT pairs:\n"); 793 794 in = NULL; 795 out = NULL; 796 list_for_each_entry(port, &tcm->dp_resources, list) { 797 if (!tb_port_is_dpin(port)) 798 continue; 799 800 if (tb_port_is_enabled(port)) { 801 tb_port_dbg(port, "in use\n"); 802 continue; 803 } 804 805 tb_port_dbg(port, "DP IN available\n"); 806 807 out = tb_find_dp_out(tb, port); 808 if (out) { 809 in = port; 810 break; 811 } 812 } 813 814 if (!in) { 815 tb_dbg(tb, "no suitable DP IN adapter available, not tunneling\n"); 816 return; 817 } 818 if (!out) { 819 tb_dbg(tb, "no suitable DP OUT adapter available, not tunneling\n"); 820 return; 821 } 822 823 if (tb_switch_alloc_dp_resource(in->sw, in)) { 824 tb_port_dbg(in, "no resource available for DP IN, not tunneling\n"); 825 return; 826 } 827 828 /* Make all unused USB3 bandwidth available for the new DP tunnel */ 829 ret = tb_release_unused_usb3_bandwidth(tb, in, out); 830 if (ret) { 831 tb_warn(tb, "failed to release unused bandwidth\n"); 832 goto err_dealloc_dp; 833 } 834 835 ret = tb_available_bandwidth(tb, in, out, &available_up, 836 &available_down); 837 if (ret) 838 goto err_reclaim; 839 840 tb_dbg(tb, "available bandwidth for new DP tunnel %u/%u Mb/s\n", 841 available_up, available_down); 842 843 tunnel = tb_tunnel_alloc_dp(tb, in, out, available_up, available_down); 844 if (!tunnel) { 845 tb_port_dbg(out, "could not allocate DP tunnel\n"); 846 goto err_reclaim; 847 } 848 849 if (tb_tunnel_activate(tunnel)) { 850 tb_port_info(out, "DP tunnel activation failed, aborting\n"); 851 goto err_free; 852 } 853 854 list_add_tail(&tunnel->list, &tcm->tunnel_list); 855 tb_reclaim_usb3_bandwidth(tb, in, out); 856 return; 857 858 err_free: 859 tb_tunnel_free(tunnel); 860 err_reclaim: 861 tb_reclaim_usb3_bandwidth(tb, in, out); 862 err_dealloc_dp: 863 tb_switch_dealloc_dp_resource(in->sw, in); 864 } 865 866 static void tb_dp_resource_unavailable(struct tb *tb, struct tb_port *port) 867 { 868 struct tb_port *in, *out; 869 struct tb_tunnel *tunnel; 870 871 if (tb_port_is_dpin(port)) { 872 tb_port_dbg(port, "DP IN resource unavailable\n"); 873 in = port; 874 out = NULL; 875 } else { 876 tb_port_dbg(port, "DP OUT resource unavailable\n"); 877 in = NULL; 878 out = port; 879 } 880 881 tunnel = tb_find_tunnel(tb, TB_TUNNEL_DP, in, out); 882 tb_deactivate_and_free_tunnel(tunnel); 883 list_del_init(&port->list); 884 885 /* 886 * See if there is another DP OUT port that can be used for 887 * to create another tunnel. 888 */ 889 tb_tunnel_dp(tb); 890 } 891 892 static void tb_dp_resource_available(struct tb *tb, struct tb_port *port) 893 { 894 struct tb_cm *tcm = tb_priv(tb); 895 struct tb_port *p; 896 897 if (tb_port_is_enabled(port)) 898 return; 899 900 list_for_each_entry(p, &tcm->dp_resources, list) { 901 if (p == port) 902 return; 903 } 904 905 tb_port_dbg(port, "DP %s resource available\n", 906 tb_port_is_dpin(port) ? "IN" : "OUT"); 907 list_add_tail(&port->list, &tcm->dp_resources); 908 909 /* Look for suitable DP IN <-> DP OUT pairs now */ 910 tb_tunnel_dp(tb); 911 } 912 913 static void tb_disconnect_and_release_dp(struct tb *tb) 914 { 915 struct tb_cm *tcm = tb_priv(tb); 916 struct tb_tunnel *tunnel, *n; 917 918 /* 919 * Tear down all DP tunnels and release their resources. They 920 * will be re-established after resume based on plug events. 921 */ 922 list_for_each_entry_safe_reverse(tunnel, n, &tcm->tunnel_list, list) { 923 if (tb_tunnel_is_dp(tunnel)) 924 tb_deactivate_and_free_tunnel(tunnel); 925 } 926 927 while (!list_empty(&tcm->dp_resources)) { 928 struct tb_port *port; 929 930 port = list_first_entry(&tcm->dp_resources, 931 struct tb_port, list); 932 list_del_init(&port->list); 933 } 934 } 935 936 static int tb_tunnel_pci(struct tb *tb, struct tb_switch *sw) 937 { 938 struct tb_port *up, *down, *port; 939 struct tb_cm *tcm = tb_priv(tb); 940 struct tb_switch *parent_sw; 941 struct tb_tunnel *tunnel; 942 943 up = tb_switch_find_port(sw, TB_TYPE_PCIE_UP); 944 if (!up) 945 return 0; 946 947 /* 948 * Look up available down port. Since we are chaining it should 949 * be found right above this switch. 950 */ 951 parent_sw = tb_to_switch(sw->dev.parent); 952 port = tb_port_at(tb_route(sw), parent_sw); 953 down = tb_find_pcie_down(parent_sw, port); 954 if (!down) 955 return 0; 956 957 tunnel = tb_tunnel_alloc_pci(tb, up, down); 958 if (!tunnel) 959 return -ENOMEM; 960 961 if (tb_tunnel_activate(tunnel)) { 962 tb_port_info(up, 963 "PCIe tunnel activation failed, aborting\n"); 964 tb_tunnel_free(tunnel); 965 return -EIO; 966 } 967 968 list_add_tail(&tunnel->list, &tcm->tunnel_list); 969 return 0; 970 } 971 972 static int tb_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd) 973 { 974 struct tb_cm *tcm = tb_priv(tb); 975 struct tb_port *nhi_port, *dst_port; 976 struct tb_tunnel *tunnel; 977 struct tb_switch *sw; 978 979 sw = tb_to_switch(xd->dev.parent); 980 dst_port = tb_port_at(xd->route, sw); 981 nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI); 982 983 mutex_lock(&tb->lock); 984 tunnel = tb_tunnel_alloc_dma(tb, nhi_port, dst_port, xd->transmit_ring, 985 xd->transmit_path, xd->receive_ring, 986 xd->receive_path); 987 if (!tunnel) { 988 mutex_unlock(&tb->lock); 989 return -ENOMEM; 990 } 991 992 if (tb_tunnel_activate(tunnel)) { 993 tb_port_info(nhi_port, 994 "DMA tunnel activation failed, aborting\n"); 995 tb_tunnel_free(tunnel); 996 mutex_unlock(&tb->lock); 997 return -EIO; 998 } 999 1000 list_add_tail(&tunnel->list, &tcm->tunnel_list); 1001 mutex_unlock(&tb->lock); 1002 return 0; 1003 } 1004 1005 static void __tb_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd) 1006 { 1007 struct tb_port *dst_port; 1008 struct tb_tunnel *tunnel; 1009 struct tb_switch *sw; 1010 1011 sw = tb_to_switch(xd->dev.parent); 1012 dst_port = tb_port_at(xd->route, sw); 1013 1014 /* 1015 * It is possible that the tunnel was already teared down (in 1016 * case of cable disconnect) so it is fine if we cannot find it 1017 * here anymore. 1018 */ 1019 tunnel = tb_find_tunnel(tb, TB_TUNNEL_DMA, NULL, dst_port); 1020 tb_deactivate_and_free_tunnel(tunnel); 1021 } 1022 1023 static int tb_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd) 1024 { 1025 if (!xd->is_unplugged) { 1026 mutex_lock(&tb->lock); 1027 __tb_disconnect_xdomain_paths(tb, xd); 1028 mutex_unlock(&tb->lock); 1029 } 1030 return 0; 1031 } 1032 1033 /* hotplug handling */ 1034 1035 /** 1036 * tb_handle_hotplug() - handle hotplug event 1037 * 1038 * Executes on tb->wq. 1039 */ 1040 static void tb_handle_hotplug(struct work_struct *work) 1041 { 1042 struct tb_hotplug_event *ev = container_of(work, typeof(*ev), work); 1043 struct tb *tb = ev->tb; 1044 struct tb_cm *tcm = tb_priv(tb); 1045 struct tb_switch *sw; 1046 struct tb_port *port; 1047 mutex_lock(&tb->lock); 1048 if (!tcm->hotplug_active) 1049 goto out; /* during init, suspend or shutdown */ 1050 1051 sw = tb_switch_find_by_route(tb, ev->route); 1052 if (!sw) { 1053 tb_warn(tb, 1054 "hotplug event from non existent switch %llx:%x (unplug: %d)\n", 1055 ev->route, ev->port, ev->unplug); 1056 goto out; 1057 } 1058 if (ev->port > sw->config.max_port_number) { 1059 tb_warn(tb, 1060 "hotplug event from non existent port %llx:%x (unplug: %d)\n", 1061 ev->route, ev->port, ev->unplug); 1062 goto put_sw; 1063 } 1064 port = &sw->ports[ev->port]; 1065 if (tb_is_upstream_port(port)) { 1066 tb_dbg(tb, "hotplug event for upstream port %llx:%x (unplug: %d)\n", 1067 ev->route, ev->port, ev->unplug); 1068 goto put_sw; 1069 } 1070 if (ev->unplug) { 1071 tb_retimer_remove_all(port); 1072 1073 if (tb_port_has_remote(port)) { 1074 tb_port_dbg(port, "switch unplugged\n"); 1075 tb_sw_set_unplugged(port->remote->sw); 1076 tb_free_invalid_tunnels(tb); 1077 tb_remove_dp_resources(port->remote->sw); 1078 tb_switch_tmu_disable(port->remote->sw); 1079 tb_switch_lane_bonding_disable(port->remote->sw); 1080 tb_switch_remove(port->remote->sw); 1081 port->remote = NULL; 1082 if (port->dual_link_port) 1083 port->dual_link_port->remote = NULL; 1084 /* Maybe we can create another DP tunnel */ 1085 tb_tunnel_dp(tb); 1086 } else if (port->xdomain) { 1087 struct tb_xdomain *xd = tb_xdomain_get(port->xdomain); 1088 1089 tb_port_dbg(port, "xdomain unplugged\n"); 1090 /* 1091 * Service drivers are unbound during 1092 * tb_xdomain_remove() so setting XDomain as 1093 * unplugged here prevents deadlock if they call 1094 * tb_xdomain_disable_paths(). We will tear down 1095 * the path below. 1096 */ 1097 xd->is_unplugged = true; 1098 tb_xdomain_remove(xd); 1099 port->xdomain = NULL; 1100 __tb_disconnect_xdomain_paths(tb, xd); 1101 tb_xdomain_put(xd); 1102 } else if (tb_port_is_dpout(port) || tb_port_is_dpin(port)) { 1103 tb_dp_resource_unavailable(tb, port); 1104 } else { 1105 tb_port_dbg(port, 1106 "got unplug event for disconnected port, ignoring\n"); 1107 } 1108 } else if (port->remote) { 1109 tb_port_dbg(port, "got plug event for connected port, ignoring\n"); 1110 } else { 1111 if (tb_port_is_null(port)) { 1112 tb_port_dbg(port, "hotplug: scanning\n"); 1113 tb_scan_port(port); 1114 if (!port->remote) 1115 tb_port_dbg(port, "hotplug: no switch found\n"); 1116 } else if (tb_port_is_dpout(port) || tb_port_is_dpin(port)) { 1117 tb_dp_resource_available(tb, port); 1118 } 1119 } 1120 1121 put_sw: 1122 tb_switch_put(sw); 1123 out: 1124 mutex_unlock(&tb->lock); 1125 kfree(ev); 1126 } 1127 1128 /** 1129 * tb_schedule_hotplug_handler() - callback function for the control channel 1130 * 1131 * Delegates to tb_handle_hotplug. 1132 */ 1133 static void tb_handle_event(struct tb *tb, enum tb_cfg_pkg_type type, 1134 const void *buf, size_t size) 1135 { 1136 const struct cfg_event_pkg *pkg = buf; 1137 u64 route; 1138 1139 if (type != TB_CFG_PKG_EVENT) { 1140 tb_warn(tb, "unexpected event %#x, ignoring\n", type); 1141 return; 1142 } 1143 1144 route = tb_cfg_get_route(&pkg->header); 1145 1146 if (tb_cfg_ack_plug(tb->ctl, route, pkg->port, pkg->unplug)) { 1147 tb_warn(tb, "could not ack plug event on %llx:%x\n", route, 1148 pkg->port); 1149 } 1150 1151 tb_queue_hotplug(tb, route, pkg->port, pkg->unplug); 1152 } 1153 1154 static void tb_stop(struct tb *tb) 1155 { 1156 struct tb_cm *tcm = tb_priv(tb); 1157 struct tb_tunnel *tunnel; 1158 struct tb_tunnel *n; 1159 1160 /* tunnels are only present after everything has been initialized */ 1161 list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) { 1162 /* 1163 * DMA tunnels require the driver to be functional so we 1164 * tear them down. Other protocol tunnels can be left 1165 * intact. 1166 */ 1167 if (tb_tunnel_is_dma(tunnel)) 1168 tb_tunnel_deactivate(tunnel); 1169 tb_tunnel_free(tunnel); 1170 } 1171 tb_switch_remove(tb->root_switch); 1172 tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */ 1173 } 1174 1175 static int tb_scan_finalize_switch(struct device *dev, void *data) 1176 { 1177 if (tb_is_switch(dev)) { 1178 struct tb_switch *sw = tb_to_switch(dev); 1179 1180 /* 1181 * If we found that the switch was already setup by the 1182 * boot firmware, mark it as authorized now before we 1183 * send uevent to userspace. 1184 */ 1185 if (sw->boot) 1186 sw->authorized = 1; 1187 1188 dev_set_uevent_suppress(dev, false); 1189 kobject_uevent(&dev->kobj, KOBJ_ADD); 1190 device_for_each_child(dev, NULL, tb_scan_finalize_switch); 1191 } 1192 1193 return 0; 1194 } 1195 1196 static int tb_start(struct tb *tb) 1197 { 1198 struct tb_cm *tcm = tb_priv(tb); 1199 int ret; 1200 1201 tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0); 1202 if (IS_ERR(tb->root_switch)) 1203 return PTR_ERR(tb->root_switch); 1204 1205 /* 1206 * ICM firmware upgrade needs running firmware and in native 1207 * mode that is not available so disable firmware upgrade of the 1208 * root switch. 1209 */ 1210 tb->root_switch->no_nvm_upgrade = true; 1211 1212 ret = tb_switch_configure(tb->root_switch); 1213 if (ret) { 1214 tb_switch_put(tb->root_switch); 1215 return ret; 1216 } 1217 1218 /* Announce the switch to the world */ 1219 ret = tb_switch_add(tb->root_switch); 1220 if (ret) { 1221 tb_switch_put(tb->root_switch); 1222 return ret; 1223 } 1224 1225 /* Enable TMU if it is off */ 1226 tb_switch_tmu_enable(tb->root_switch); 1227 /* Full scan to discover devices added before the driver was loaded. */ 1228 tb_scan_switch(tb->root_switch); 1229 /* Find out tunnels created by the boot firmware */ 1230 tb_discover_tunnels(tb->root_switch); 1231 /* 1232 * If the boot firmware did not create USB 3.x tunnels create them 1233 * now for the whole topology. 1234 */ 1235 tb_create_usb3_tunnels(tb->root_switch); 1236 /* Add DP IN resources for the root switch */ 1237 tb_add_dp_resources(tb->root_switch); 1238 /* Make the discovered switches available to the userspace */ 1239 device_for_each_child(&tb->root_switch->dev, NULL, 1240 tb_scan_finalize_switch); 1241 1242 /* Allow tb_handle_hotplug to progress events */ 1243 tcm->hotplug_active = true; 1244 return 0; 1245 } 1246 1247 static int tb_suspend_noirq(struct tb *tb) 1248 { 1249 struct tb_cm *tcm = tb_priv(tb); 1250 1251 tb_dbg(tb, "suspending...\n"); 1252 tb_disconnect_and_release_dp(tb); 1253 tb_switch_suspend(tb->root_switch); 1254 tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */ 1255 tb_dbg(tb, "suspend finished\n"); 1256 1257 return 0; 1258 } 1259 1260 static void tb_restore_children(struct tb_switch *sw) 1261 { 1262 struct tb_port *port; 1263 1264 if (tb_enable_tmu(sw)) 1265 tb_sw_warn(sw, "failed to restore TMU configuration\n"); 1266 1267 tb_switch_for_each_port(sw, port) { 1268 if (!tb_port_has_remote(port)) 1269 continue; 1270 1271 tb_switch_lane_bonding_enable(port->remote->sw); 1272 1273 tb_restore_children(port->remote->sw); 1274 } 1275 } 1276 1277 static int tb_resume_noirq(struct tb *tb) 1278 { 1279 struct tb_cm *tcm = tb_priv(tb); 1280 struct tb_tunnel *tunnel, *n; 1281 1282 tb_dbg(tb, "resuming...\n"); 1283 1284 /* remove any pci devices the firmware might have setup */ 1285 tb_switch_reset(tb->root_switch); 1286 1287 tb_switch_resume(tb->root_switch); 1288 tb_free_invalid_tunnels(tb); 1289 tb_free_unplugged_children(tb->root_switch); 1290 tb_restore_children(tb->root_switch); 1291 list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) 1292 tb_tunnel_restart(tunnel); 1293 if (!list_empty(&tcm->tunnel_list)) { 1294 /* 1295 * the pcie links need some time to get going. 1296 * 100ms works for me... 1297 */ 1298 tb_dbg(tb, "tunnels restarted, sleeping for 100ms\n"); 1299 msleep(100); 1300 } 1301 /* Allow tb_handle_hotplug to progress events */ 1302 tcm->hotplug_active = true; 1303 tb_dbg(tb, "resume finished\n"); 1304 1305 return 0; 1306 } 1307 1308 static int tb_free_unplugged_xdomains(struct tb_switch *sw) 1309 { 1310 struct tb_port *port; 1311 int ret = 0; 1312 1313 tb_switch_for_each_port(sw, port) { 1314 if (tb_is_upstream_port(port)) 1315 continue; 1316 if (port->xdomain && port->xdomain->is_unplugged) { 1317 tb_retimer_remove_all(port); 1318 tb_xdomain_remove(port->xdomain); 1319 port->xdomain = NULL; 1320 ret++; 1321 } else if (port->remote) { 1322 ret += tb_free_unplugged_xdomains(port->remote->sw); 1323 } 1324 } 1325 1326 return ret; 1327 } 1328 1329 static void tb_complete(struct tb *tb) 1330 { 1331 /* 1332 * Release any unplugged XDomains and if there is a case where 1333 * another domain is swapped in place of unplugged XDomain we 1334 * need to run another rescan. 1335 */ 1336 mutex_lock(&tb->lock); 1337 if (tb_free_unplugged_xdomains(tb->root_switch)) 1338 tb_scan_switch(tb->root_switch); 1339 mutex_unlock(&tb->lock); 1340 } 1341 1342 static const struct tb_cm_ops tb_cm_ops = { 1343 .start = tb_start, 1344 .stop = tb_stop, 1345 .suspend_noirq = tb_suspend_noirq, 1346 .resume_noirq = tb_resume_noirq, 1347 .complete = tb_complete, 1348 .handle_event = tb_handle_event, 1349 .approve_switch = tb_tunnel_pci, 1350 .approve_xdomain_paths = tb_approve_xdomain_paths, 1351 .disconnect_xdomain_paths = tb_disconnect_xdomain_paths, 1352 }; 1353 1354 struct tb *tb_probe(struct tb_nhi *nhi) 1355 { 1356 struct tb_cm *tcm; 1357 struct tb *tb; 1358 1359 tb = tb_domain_alloc(nhi, sizeof(*tcm)); 1360 if (!tb) 1361 return NULL; 1362 1363 tb->security_level = TB_SECURITY_USER; 1364 tb->cm_ops = &tb_cm_ops; 1365 1366 tcm = tb_priv(tb); 1367 INIT_LIST_HEAD(&tcm->tunnel_list); 1368 INIT_LIST_HEAD(&tcm->dp_resources); 1369 1370 return tb; 1371 } 1372