1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * drivers/base/dd.c - The core device/driver interactions. 4 * 5 * This file contains the (sometimes tricky) code that controls the 6 * interactions between devices and drivers, which primarily includes 7 * driver binding and unbinding. 8 * 9 * All of this code used to exist in drivers/base/bus.c, but was 10 * relocated to here in the name of compartmentalization (since it wasn't 11 * strictly code just for the 'struct bus_type'. 12 * 13 * Copyright (c) 2002-5 Patrick Mochel 14 * Copyright (c) 2002-3 Open Source Development Labs 15 * Copyright (c) 2007-2009 Greg Kroah-Hartman <gregkh@suse.de> 16 * Copyright (c) 2007-2009 Novell Inc. 17 */ 18 19 #include <linux/debugfs.h> 20 #include <linux/device.h> 21 #include <linux/delay.h> 22 #include <linux/dma-map-ops.h> 23 #include <linux/init.h> 24 #include <linux/module.h> 25 #include <linux/kthread.h> 26 #include <linux/wait.h> 27 #include <linux/async.h> 28 #include <linux/pm_runtime.h> 29 #include <linux/pinctrl/devinfo.h> 30 #include <linux/slab.h> 31 32 #include "base.h" 33 #include "power/power.h" 34 35 /* 36 * Deferred Probe infrastructure. 37 * 38 * Sometimes driver probe order matters, but the kernel doesn't always have 39 * dependency information which means some drivers will get probed before a 40 * resource it depends on is available. For example, an SDHCI driver may 41 * first need a GPIO line from an i2c GPIO controller before it can be 42 * initialized. If a required resource is not available yet, a driver can 43 * request probing to be deferred by returning -EPROBE_DEFER from its probe hook 44 * 45 * Deferred probe maintains two lists of devices, a pending list and an active 46 * list. A driver returning -EPROBE_DEFER causes the device to be added to the 47 * pending list. A successful driver probe will trigger moving all devices 48 * from the pending to the active list so that the workqueue will eventually 49 * retry them. 50 * 51 * The deferred_probe_mutex must be held any time the deferred_probe_*_list 52 * of the (struct device*)->p->deferred_probe pointers are manipulated 53 */ 54 static DEFINE_MUTEX(deferred_probe_mutex); 55 static LIST_HEAD(deferred_probe_pending_list); 56 static LIST_HEAD(deferred_probe_active_list); 57 static atomic_t deferred_trigger_count = ATOMIC_INIT(0); 58 static bool initcalls_done; 59 60 /* Save the async probe drivers' name from kernel cmdline */ 61 #define ASYNC_DRV_NAMES_MAX_LEN 256 62 static char async_probe_drv_names[ASYNC_DRV_NAMES_MAX_LEN]; 63 64 /* 65 * In some cases, like suspend to RAM or hibernation, It might be reasonable 66 * to prohibit probing of devices as it could be unsafe. 67 * Once defer_all_probes is true all drivers probes will be forcibly deferred. 68 */ 69 static bool defer_all_probes; 70 71 static void __device_set_deferred_probe_reason(const struct device *dev, char *reason) 72 { 73 kfree(dev->p->deferred_probe_reason); 74 dev->p->deferred_probe_reason = reason; 75 } 76 77 /* 78 * deferred_probe_work_func() - Retry probing devices in the active list. 79 */ 80 static void deferred_probe_work_func(struct work_struct *work) 81 { 82 struct device *dev; 83 struct device_private *private; 84 /* 85 * This block processes every device in the deferred 'active' list. 86 * Each device is removed from the active list and passed to 87 * bus_probe_device() to re-attempt the probe. The loop continues 88 * until every device in the active list is removed and retried. 89 * 90 * Note: Once the device is removed from the list and the mutex is 91 * released, it is possible for the device get freed by another thread 92 * and cause a illegal pointer dereference. This code uses 93 * get/put_device() to ensure the device structure cannot disappear 94 * from under our feet. 95 */ 96 mutex_lock(&deferred_probe_mutex); 97 while (!list_empty(&deferred_probe_active_list)) { 98 private = list_first_entry(&deferred_probe_active_list, 99 typeof(*dev->p), deferred_probe); 100 dev = private->device; 101 list_del_init(&private->deferred_probe); 102 103 get_device(dev); 104 105 __device_set_deferred_probe_reason(dev, NULL); 106 107 /* 108 * Drop the mutex while probing each device; the probe path may 109 * manipulate the deferred list 110 */ 111 mutex_unlock(&deferred_probe_mutex); 112 113 /* 114 * Force the device to the end of the dpm_list since 115 * the PM code assumes that the order we add things to 116 * the list is a good order for suspend but deferred 117 * probe makes that very unsafe. 118 */ 119 device_pm_move_to_tail(dev); 120 121 dev_dbg(dev, "Retrying from deferred list\n"); 122 bus_probe_device(dev); 123 mutex_lock(&deferred_probe_mutex); 124 125 put_device(dev); 126 } 127 mutex_unlock(&deferred_probe_mutex); 128 } 129 static DECLARE_WORK(deferred_probe_work, deferred_probe_work_func); 130 131 void driver_deferred_probe_add(struct device *dev) 132 { 133 if (!dev->can_match) 134 return; 135 136 mutex_lock(&deferred_probe_mutex); 137 if (list_empty(&dev->p->deferred_probe)) { 138 dev_dbg(dev, "Added to deferred list\n"); 139 list_add_tail(&dev->p->deferred_probe, &deferred_probe_pending_list); 140 } 141 mutex_unlock(&deferred_probe_mutex); 142 } 143 144 void driver_deferred_probe_del(struct device *dev) 145 { 146 mutex_lock(&deferred_probe_mutex); 147 if (!list_empty(&dev->p->deferred_probe)) { 148 dev_dbg(dev, "Removed from deferred list\n"); 149 list_del_init(&dev->p->deferred_probe); 150 __device_set_deferred_probe_reason(dev, NULL); 151 } 152 mutex_unlock(&deferred_probe_mutex); 153 } 154 155 static bool driver_deferred_probe_enable; 156 /** 157 * driver_deferred_probe_trigger() - Kick off re-probing deferred devices 158 * 159 * This functions moves all devices from the pending list to the active 160 * list and schedules the deferred probe workqueue to process them. It 161 * should be called anytime a driver is successfully bound to a device. 162 * 163 * Note, there is a race condition in multi-threaded probe. In the case where 164 * more than one device is probing at the same time, it is possible for one 165 * probe to complete successfully while another is about to defer. If the second 166 * depends on the first, then it will get put on the pending list after the 167 * trigger event has already occurred and will be stuck there. 168 * 169 * The atomic 'deferred_trigger_count' is used to determine if a successful 170 * trigger has occurred in the midst of probing a driver. If the trigger count 171 * changes in the midst of a probe, then deferred processing should be triggered 172 * again. 173 */ 174 static void driver_deferred_probe_trigger(void) 175 { 176 if (!driver_deferred_probe_enable) 177 return; 178 179 /* 180 * A successful probe means that all the devices in the pending list 181 * should be triggered to be reprobed. Move all the deferred devices 182 * into the active list so they can be retried by the workqueue 183 */ 184 mutex_lock(&deferred_probe_mutex); 185 atomic_inc(&deferred_trigger_count); 186 list_splice_tail_init(&deferred_probe_pending_list, 187 &deferred_probe_active_list); 188 mutex_unlock(&deferred_probe_mutex); 189 190 /* 191 * Kick the re-probe thread. It may already be scheduled, but it is 192 * safe to kick it again. 193 */ 194 queue_work(system_unbound_wq, &deferred_probe_work); 195 } 196 197 /** 198 * device_block_probing() - Block/defer device's probes 199 * 200 * It will disable probing of devices and defer their probes instead. 201 */ 202 void device_block_probing(void) 203 { 204 defer_all_probes = true; 205 /* sync with probes to avoid races. */ 206 wait_for_device_probe(); 207 } 208 209 /** 210 * device_unblock_probing() - Unblock/enable device's probes 211 * 212 * It will restore normal behavior and trigger re-probing of deferred 213 * devices. 214 */ 215 void device_unblock_probing(void) 216 { 217 defer_all_probes = false; 218 driver_deferred_probe_trigger(); 219 } 220 221 /** 222 * device_set_deferred_probe_reason() - Set defer probe reason message for device 223 * @dev: the pointer to the struct device 224 * @vaf: the pointer to va_format structure with message 225 */ 226 void device_set_deferred_probe_reason(const struct device *dev, struct va_format *vaf) 227 { 228 const char *drv = dev_driver_string(dev); 229 char *reason; 230 231 mutex_lock(&deferred_probe_mutex); 232 233 reason = kasprintf(GFP_KERNEL, "%s: %pV", drv, vaf); 234 __device_set_deferred_probe_reason(dev, reason); 235 236 mutex_unlock(&deferred_probe_mutex); 237 } 238 239 /* 240 * deferred_devs_show() - Show the devices in the deferred probe pending list. 241 */ 242 static int deferred_devs_show(struct seq_file *s, void *data) 243 { 244 struct device_private *curr; 245 246 mutex_lock(&deferred_probe_mutex); 247 248 list_for_each_entry(curr, &deferred_probe_pending_list, deferred_probe) 249 seq_printf(s, "%s\t%s", dev_name(curr->device), 250 curr->device->p->deferred_probe_reason ?: "\n"); 251 252 mutex_unlock(&deferred_probe_mutex); 253 254 return 0; 255 } 256 DEFINE_SHOW_ATTRIBUTE(deferred_devs); 257 258 int driver_deferred_probe_timeout; 259 EXPORT_SYMBOL_GPL(driver_deferred_probe_timeout); 260 static DECLARE_WAIT_QUEUE_HEAD(probe_timeout_waitqueue); 261 262 static int __init deferred_probe_timeout_setup(char *str) 263 { 264 int timeout; 265 266 if (!kstrtoint(str, 10, &timeout)) 267 driver_deferred_probe_timeout = timeout; 268 return 1; 269 } 270 __setup("deferred_probe_timeout=", deferred_probe_timeout_setup); 271 272 /** 273 * driver_deferred_probe_check_state() - Check deferred probe state 274 * @dev: device to check 275 * 276 * Return: 277 * -ENODEV if initcalls have completed and modules are disabled. 278 * -ETIMEDOUT if the deferred probe timeout was set and has expired 279 * and modules are enabled. 280 * -EPROBE_DEFER in other cases. 281 * 282 * Drivers or subsystems can opt-in to calling this function instead of directly 283 * returning -EPROBE_DEFER. 284 */ 285 int driver_deferred_probe_check_state(struct device *dev) 286 { 287 if (!IS_ENABLED(CONFIG_MODULES) && initcalls_done) { 288 dev_warn(dev, "ignoring dependency for device, assuming no driver\n"); 289 return -ENODEV; 290 } 291 292 if (!driver_deferred_probe_timeout && initcalls_done) { 293 dev_warn(dev, "deferred probe timeout, ignoring dependency\n"); 294 return -ETIMEDOUT; 295 } 296 297 return -EPROBE_DEFER; 298 } 299 EXPORT_SYMBOL_GPL(driver_deferred_probe_check_state); 300 301 static void deferred_probe_timeout_work_func(struct work_struct *work) 302 { 303 struct device_private *p; 304 305 fw_devlink_drivers_done(); 306 307 driver_deferred_probe_timeout = 0; 308 driver_deferred_probe_trigger(); 309 flush_work(&deferred_probe_work); 310 311 mutex_lock(&deferred_probe_mutex); 312 list_for_each_entry(p, &deferred_probe_pending_list, deferred_probe) 313 dev_info(p->device, "deferred probe pending\n"); 314 mutex_unlock(&deferred_probe_mutex); 315 wake_up_all(&probe_timeout_waitqueue); 316 } 317 static DECLARE_DELAYED_WORK(deferred_probe_timeout_work, deferred_probe_timeout_work_func); 318 319 /** 320 * deferred_probe_initcall() - Enable probing of deferred devices 321 * 322 * We don't want to get in the way when the bulk of drivers are getting probed. 323 * Instead, this initcall makes sure that deferred probing is delayed until 324 * late_initcall time. 325 */ 326 static int deferred_probe_initcall(void) 327 { 328 debugfs_create_file("devices_deferred", 0444, NULL, NULL, 329 &deferred_devs_fops); 330 331 driver_deferred_probe_enable = true; 332 driver_deferred_probe_trigger(); 333 /* Sort as many dependencies as possible before exiting initcalls */ 334 flush_work(&deferred_probe_work); 335 initcalls_done = true; 336 337 if (!IS_ENABLED(CONFIG_MODULES)) 338 fw_devlink_drivers_done(); 339 340 /* 341 * Trigger deferred probe again, this time we won't defer anything 342 * that is optional 343 */ 344 driver_deferred_probe_trigger(); 345 flush_work(&deferred_probe_work); 346 347 if (driver_deferred_probe_timeout > 0) { 348 schedule_delayed_work(&deferred_probe_timeout_work, 349 driver_deferred_probe_timeout * HZ); 350 } 351 return 0; 352 } 353 late_initcall(deferred_probe_initcall); 354 355 static void __exit deferred_probe_exit(void) 356 { 357 debugfs_remove_recursive(debugfs_lookup("devices_deferred", NULL)); 358 } 359 __exitcall(deferred_probe_exit); 360 361 /** 362 * device_is_bound() - Check if device is bound to a driver 363 * @dev: device to check 364 * 365 * Returns true if passed device has already finished probing successfully 366 * against a driver. 367 * 368 * This function must be called with the device lock held. 369 */ 370 bool device_is_bound(struct device *dev) 371 { 372 return dev->p && klist_node_attached(&dev->p->knode_driver); 373 } 374 375 static void driver_bound(struct device *dev) 376 { 377 if (device_is_bound(dev)) { 378 pr_warn("%s: device %s already bound\n", 379 __func__, kobject_name(&dev->kobj)); 380 return; 381 } 382 383 pr_debug("driver: '%s': %s: bound to device '%s'\n", dev->driver->name, 384 __func__, dev_name(dev)); 385 386 klist_add_tail(&dev->p->knode_driver, &dev->driver->p->klist_devices); 387 device_links_driver_bound(dev); 388 389 device_pm_check_callbacks(dev); 390 391 /* 392 * Make sure the device is no longer in one of the deferred lists and 393 * kick off retrying all pending devices 394 */ 395 driver_deferred_probe_del(dev); 396 driver_deferred_probe_trigger(); 397 398 if (dev->bus) 399 blocking_notifier_call_chain(&dev->bus->p->bus_notifier, 400 BUS_NOTIFY_BOUND_DRIVER, dev); 401 402 kobject_uevent(&dev->kobj, KOBJ_BIND); 403 } 404 405 static ssize_t coredump_store(struct device *dev, struct device_attribute *attr, 406 const char *buf, size_t count) 407 { 408 device_lock(dev); 409 dev->driver->coredump(dev); 410 device_unlock(dev); 411 412 return count; 413 } 414 static DEVICE_ATTR_WO(coredump); 415 416 static int driver_sysfs_add(struct device *dev) 417 { 418 int ret; 419 420 if (dev->bus) 421 blocking_notifier_call_chain(&dev->bus->p->bus_notifier, 422 BUS_NOTIFY_BIND_DRIVER, dev); 423 424 ret = sysfs_create_link(&dev->driver->p->kobj, &dev->kobj, 425 kobject_name(&dev->kobj)); 426 if (ret) 427 goto fail; 428 429 ret = sysfs_create_link(&dev->kobj, &dev->driver->p->kobj, 430 "driver"); 431 if (ret) 432 goto rm_dev; 433 434 if (!IS_ENABLED(CONFIG_DEV_COREDUMP) || !dev->driver->coredump) 435 return 0; 436 437 ret = device_create_file(dev, &dev_attr_coredump); 438 if (!ret) 439 return 0; 440 441 sysfs_remove_link(&dev->kobj, "driver"); 442 443 rm_dev: 444 sysfs_remove_link(&dev->driver->p->kobj, 445 kobject_name(&dev->kobj)); 446 447 fail: 448 return ret; 449 } 450 451 static void driver_sysfs_remove(struct device *dev) 452 { 453 struct device_driver *drv = dev->driver; 454 455 if (drv) { 456 if (drv->coredump) 457 device_remove_file(dev, &dev_attr_coredump); 458 sysfs_remove_link(&drv->p->kobj, kobject_name(&dev->kobj)); 459 sysfs_remove_link(&dev->kobj, "driver"); 460 } 461 } 462 463 /** 464 * device_bind_driver - bind a driver to one device. 465 * @dev: device. 466 * 467 * Allow manual attachment of a driver to a device. 468 * Caller must have already set @dev->driver. 469 * 470 * Note that this does not modify the bus reference count. 471 * Please verify that is accounted for before calling this. 472 * (It is ok to call with no other effort from a driver's probe() method.) 473 * 474 * This function must be called with the device lock held. 475 * 476 * Callers should prefer to use device_driver_attach() instead. 477 */ 478 int device_bind_driver(struct device *dev) 479 { 480 int ret; 481 482 ret = driver_sysfs_add(dev); 483 if (!ret) { 484 device_links_force_bind(dev); 485 driver_bound(dev); 486 } 487 else if (dev->bus) 488 blocking_notifier_call_chain(&dev->bus->p->bus_notifier, 489 BUS_NOTIFY_DRIVER_NOT_BOUND, dev); 490 return ret; 491 } 492 EXPORT_SYMBOL_GPL(device_bind_driver); 493 494 static atomic_t probe_count = ATOMIC_INIT(0); 495 static DECLARE_WAIT_QUEUE_HEAD(probe_waitqueue); 496 497 static ssize_t state_synced_show(struct device *dev, 498 struct device_attribute *attr, char *buf) 499 { 500 bool val; 501 502 device_lock(dev); 503 val = dev->state_synced; 504 device_unlock(dev); 505 506 return sysfs_emit(buf, "%u\n", val); 507 } 508 static DEVICE_ATTR_RO(state_synced); 509 510 static void device_unbind_cleanup(struct device *dev) 511 { 512 devres_release_all(dev); 513 arch_teardown_dma_ops(dev); 514 kfree(dev->dma_range_map); 515 dev->dma_range_map = NULL; 516 dev->driver = NULL; 517 dev_set_drvdata(dev, NULL); 518 if (dev->pm_domain && dev->pm_domain->dismiss) 519 dev->pm_domain->dismiss(dev); 520 pm_runtime_reinit(dev); 521 dev_pm_set_driver_flags(dev, 0); 522 } 523 524 static void device_remove(struct device *dev) 525 { 526 device_remove_file(dev, &dev_attr_state_synced); 527 device_remove_groups(dev, dev->driver->dev_groups); 528 529 if (dev->bus && dev->bus->remove) 530 dev->bus->remove(dev); 531 else if (dev->driver->remove) 532 dev->driver->remove(dev); 533 } 534 535 static int call_driver_probe(struct device *dev, struct device_driver *drv) 536 { 537 int ret = 0; 538 539 if (dev->bus->probe) 540 ret = dev->bus->probe(dev); 541 else if (drv->probe) 542 ret = drv->probe(dev); 543 544 switch (ret) { 545 case 0: 546 break; 547 case -EPROBE_DEFER: 548 /* Driver requested deferred probing */ 549 dev_dbg(dev, "Driver %s requests probe deferral\n", drv->name); 550 break; 551 case -ENODEV: 552 case -ENXIO: 553 pr_debug("%s: probe of %s rejects match %d\n", 554 drv->name, dev_name(dev), ret); 555 break; 556 default: 557 /* driver matched but the probe failed */ 558 pr_warn("%s: probe of %s failed with error %d\n", 559 drv->name, dev_name(dev), ret); 560 break; 561 } 562 563 return ret; 564 } 565 566 static int really_probe(struct device *dev, struct device_driver *drv) 567 { 568 bool test_remove = IS_ENABLED(CONFIG_DEBUG_TEST_DRIVER_REMOVE) && 569 !drv->suppress_bind_attrs; 570 int ret; 571 572 if (defer_all_probes) { 573 /* 574 * Value of defer_all_probes can be set only by 575 * device_block_probing() which, in turn, will call 576 * wait_for_device_probe() right after that to avoid any races. 577 */ 578 dev_dbg(dev, "Driver %s force probe deferral\n", drv->name); 579 return -EPROBE_DEFER; 580 } 581 582 ret = device_links_check_suppliers(dev); 583 if (ret) 584 return ret; 585 586 pr_debug("bus: '%s': %s: probing driver %s with device %s\n", 587 drv->bus->name, __func__, drv->name, dev_name(dev)); 588 if (!list_empty(&dev->devres_head)) { 589 dev_crit(dev, "Resources present before probing\n"); 590 ret = -EBUSY; 591 goto done; 592 } 593 594 re_probe: 595 dev->driver = drv; 596 597 /* If using pinctrl, bind pins now before probing */ 598 ret = pinctrl_bind_pins(dev); 599 if (ret) 600 goto pinctrl_bind_failed; 601 602 if (dev->bus->dma_configure) { 603 ret = dev->bus->dma_configure(dev); 604 if (ret) 605 goto pinctrl_bind_failed; 606 } 607 608 ret = driver_sysfs_add(dev); 609 if (ret) { 610 pr_err("%s: driver_sysfs_add(%s) failed\n", 611 __func__, dev_name(dev)); 612 goto sysfs_failed; 613 } 614 615 if (dev->pm_domain && dev->pm_domain->activate) { 616 ret = dev->pm_domain->activate(dev); 617 if (ret) 618 goto probe_failed; 619 } 620 621 ret = call_driver_probe(dev, drv); 622 if (ret) { 623 /* 624 * Return probe errors as positive values so that the callers 625 * can distinguish them from other errors. 626 */ 627 ret = -ret; 628 goto probe_failed; 629 } 630 631 ret = device_add_groups(dev, drv->dev_groups); 632 if (ret) { 633 dev_err(dev, "device_add_groups() failed\n"); 634 goto dev_groups_failed; 635 } 636 637 if (dev_has_sync_state(dev)) { 638 ret = device_create_file(dev, &dev_attr_state_synced); 639 if (ret) { 640 dev_err(dev, "state_synced sysfs add failed\n"); 641 goto dev_sysfs_state_synced_failed; 642 } 643 } 644 645 if (test_remove) { 646 test_remove = false; 647 648 device_remove(dev); 649 driver_sysfs_remove(dev); 650 device_unbind_cleanup(dev); 651 652 goto re_probe; 653 } 654 655 pinctrl_init_done(dev); 656 657 if (dev->pm_domain && dev->pm_domain->sync) 658 dev->pm_domain->sync(dev); 659 660 driver_bound(dev); 661 pr_debug("bus: '%s': %s: bound device %s to driver %s\n", 662 drv->bus->name, __func__, dev_name(dev), drv->name); 663 goto done; 664 665 dev_sysfs_state_synced_failed: 666 dev_groups_failed: 667 device_remove(dev); 668 probe_failed: 669 driver_sysfs_remove(dev); 670 sysfs_failed: 671 if (dev->bus) 672 blocking_notifier_call_chain(&dev->bus->p->bus_notifier, 673 BUS_NOTIFY_DRIVER_NOT_BOUND, dev); 674 if (dev->bus && dev->bus->dma_cleanup) 675 dev->bus->dma_cleanup(dev); 676 pinctrl_bind_failed: 677 device_links_no_driver(dev); 678 device_unbind_cleanup(dev); 679 done: 680 return ret; 681 } 682 683 /* 684 * For initcall_debug, show the driver probe time. 685 */ 686 static int really_probe_debug(struct device *dev, struct device_driver *drv) 687 { 688 ktime_t calltime, rettime; 689 int ret; 690 691 calltime = ktime_get(); 692 ret = really_probe(dev, drv); 693 rettime = ktime_get(); 694 pr_debug("probe of %s returned %d after %lld usecs\n", 695 dev_name(dev), ret, ktime_us_delta(rettime, calltime)); 696 return ret; 697 } 698 699 /** 700 * driver_probe_done 701 * Determine if the probe sequence is finished or not. 702 * 703 * Should somehow figure out how to use a semaphore, not an atomic variable... 704 */ 705 int driver_probe_done(void) 706 { 707 int local_probe_count = atomic_read(&probe_count); 708 709 pr_debug("%s: probe_count = %d\n", __func__, local_probe_count); 710 if (local_probe_count) 711 return -EBUSY; 712 return 0; 713 } 714 715 /** 716 * wait_for_device_probe 717 * Wait for device probing to be completed. 718 */ 719 void wait_for_device_probe(void) 720 { 721 /* wait for probe timeout */ 722 wait_event(probe_timeout_waitqueue, !driver_deferred_probe_timeout); 723 724 /* wait for the deferred probe workqueue to finish */ 725 flush_work(&deferred_probe_work); 726 727 /* wait for the known devices to complete their probing */ 728 wait_event(probe_waitqueue, atomic_read(&probe_count) == 0); 729 async_synchronize_full(); 730 } 731 EXPORT_SYMBOL_GPL(wait_for_device_probe); 732 733 static int __driver_probe_device(struct device_driver *drv, struct device *dev) 734 { 735 int ret = 0; 736 737 if (dev->p->dead || !device_is_registered(dev)) 738 return -ENODEV; 739 if (dev->driver) 740 return -EBUSY; 741 742 dev->can_match = true; 743 pr_debug("bus: '%s': %s: matched device %s with driver %s\n", 744 drv->bus->name, __func__, dev_name(dev), drv->name); 745 746 pm_runtime_get_suppliers(dev); 747 if (dev->parent) 748 pm_runtime_get_sync(dev->parent); 749 750 pm_runtime_barrier(dev); 751 if (initcall_debug) 752 ret = really_probe_debug(dev, drv); 753 else 754 ret = really_probe(dev, drv); 755 pm_request_idle(dev); 756 757 if (dev->parent) 758 pm_runtime_put(dev->parent); 759 760 pm_runtime_put_suppliers(dev); 761 return ret; 762 } 763 764 /** 765 * driver_probe_device - attempt to bind device & driver together 766 * @drv: driver to bind a device to 767 * @dev: device to try to bind to the driver 768 * 769 * This function returns -ENODEV if the device is not registered, -EBUSY if it 770 * already has a driver, 0 if the device is bound successfully and a positive 771 * (inverted) error code for failures from the ->probe method. 772 * 773 * This function must be called with @dev lock held. When called for a 774 * USB interface, @dev->parent lock must be held as well. 775 * 776 * If the device has a parent, runtime-resume the parent before driver probing. 777 */ 778 static int driver_probe_device(struct device_driver *drv, struct device *dev) 779 { 780 int trigger_count = atomic_read(&deferred_trigger_count); 781 int ret; 782 783 atomic_inc(&probe_count); 784 ret = __driver_probe_device(drv, dev); 785 if (ret == -EPROBE_DEFER || ret == EPROBE_DEFER) { 786 driver_deferred_probe_add(dev); 787 788 /* 789 * Did a trigger occur while probing? Need to re-trigger if yes 790 */ 791 if (trigger_count != atomic_read(&deferred_trigger_count) && 792 !defer_all_probes) 793 driver_deferred_probe_trigger(); 794 } 795 atomic_dec(&probe_count); 796 wake_up_all(&probe_waitqueue); 797 return ret; 798 } 799 800 static inline bool cmdline_requested_async_probing(const char *drv_name) 801 { 802 return parse_option_str(async_probe_drv_names, drv_name); 803 } 804 805 /* The option format is "driver_async_probe=drv_name1,drv_name2,..." */ 806 static int __init save_async_options(char *buf) 807 { 808 if (strlen(buf) >= ASYNC_DRV_NAMES_MAX_LEN) 809 pr_warn("Too long list of driver names for 'driver_async_probe'!\n"); 810 811 strlcpy(async_probe_drv_names, buf, ASYNC_DRV_NAMES_MAX_LEN); 812 return 1; 813 } 814 __setup("driver_async_probe=", save_async_options); 815 816 bool driver_allows_async_probing(struct device_driver *drv) 817 { 818 switch (drv->probe_type) { 819 case PROBE_PREFER_ASYNCHRONOUS: 820 return true; 821 822 case PROBE_FORCE_SYNCHRONOUS: 823 return false; 824 825 default: 826 if (cmdline_requested_async_probing(drv->name)) 827 return true; 828 829 if (module_requested_async_probing(drv->owner)) 830 return true; 831 832 return false; 833 } 834 } 835 836 struct device_attach_data { 837 struct device *dev; 838 839 /* 840 * Indicates whether we are considering asynchronous probing or 841 * not. Only initial binding after device or driver registration 842 * (including deferral processing) may be done asynchronously, the 843 * rest is always synchronous, as we expect it is being done by 844 * request from userspace. 845 */ 846 bool check_async; 847 848 /* 849 * Indicates if we are binding synchronous or asynchronous drivers. 850 * When asynchronous probing is enabled we'll execute 2 passes 851 * over drivers: first pass doing synchronous probing and second 852 * doing asynchronous probing (if synchronous did not succeed - 853 * most likely because there was no driver requiring synchronous 854 * probing - and we found asynchronous driver during first pass). 855 * The 2 passes are done because we can't shoot asynchronous 856 * probe for given device and driver from bus_for_each_drv() since 857 * driver pointer is not guaranteed to stay valid once 858 * bus_for_each_drv() iterates to the next driver on the bus. 859 */ 860 bool want_async; 861 862 /* 863 * We'll set have_async to 'true' if, while scanning for matching 864 * driver, we'll encounter one that requests asynchronous probing. 865 */ 866 bool have_async; 867 }; 868 869 static int __device_attach_driver(struct device_driver *drv, void *_data) 870 { 871 struct device_attach_data *data = _data; 872 struct device *dev = data->dev; 873 bool async_allowed; 874 int ret; 875 876 ret = driver_match_device(drv, dev); 877 if (ret == 0) { 878 /* no match */ 879 return 0; 880 } else if (ret == -EPROBE_DEFER) { 881 dev_dbg(dev, "Device match requests probe deferral\n"); 882 dev->can_match = true; 883 driver_deferred_probe_add(dev); 884 } else if (ret < 0) { 885 dev_dbg(dev, "Bus failed to match device: %d\n", ret); 886 return ret; 887 } /* ret > 0 means positive match */ 888 889 async_allowed = driver_allows_async_probing(drv); 890 891 if (async_allowed) 892 data->have_async = true; 893 894 if (data->check_async && async_allowed != data->want_async) 895 return 0; 896 897 /* 898 * Ignore errors returned by ->probe so that the next driver can try 899 * its luck. 900 */ 901 ret = driver_probe_device(drv, dev); 902 if (ret < 0) 903 return ret; 904 return ret == 0; 905 } 906 907 static void __device_attach_async_helper(void *_dev, async_cookie_t cookie) 908 { 909 struct device *dev = _dev; 910 struct device_attach_data data = { 911 .dev = dev, 912 .check_async = true, 913 .want_async = true, 914 }; 915 916 device_lock(dev); 917 918 /* 919 * Check if device has already been removed or claimed. This may 920 * happen with driver loading, device discovery/registration, 921 * and deferred probe processing happens all at once with 922 * multiple threads. 923 */ 924 if (dev->p->dead || dev->driver) 925 goto out_unlock; 926 927 if (dev->parent) 928 pm_runtime_get_sync(dev->parent); 929 930 bus_for_each_drv(dev->bus, NULL, &data, __device_attach_driver); 931 dev_dbg(dev, "async probe completed\n"); 932 933 pm_request_idle(dev); 934 935 if (dev->parent) 936 pm_runtime_put(dev->parent); 937 out_unlock: 938 device_unlock(dev); 939 940 put_device(dev); 941 } 942 943 static int __device_attach(struct device *dev, bool allow_async) 944 { 945 int ret = 0; 946 947 device_lock(dev); 948 if (dev->p->dead) { 949 goto out_unlock; 950 } else if (dev->driver) { 951 if (device_is_bound(dev)) { 952 ret = 1; 953 goto out_unlock; 954 } 955 ret = device_bind_driver(dev); 956 if (ret == 0) 957 ret = 1; 958 else { 959 dev->driver = NULL; 960 ret = 0; 961 } 962 } else { 963 struct device_attach_data data = { 964 .dev = dev, 965 .check_async = allow_async, 966 .want_async = false, 967 }; 968 969 if (dev->parent) 970 pm_runtime_get_sync(dev->parent); 971 972 ret = bus_for_each_drv(dev->bus, NULL, &data, 973 __device_attach_driver); 974 if (!ret && allow_async && data.have_async) { 975 /* 976 * If we could not find appropriate driver 977 * synchronously and we are allowed to do 978 * async probes and there are drivers that 979 * want to probe asynchronously, we'll 980 * try them. 981 */ 982 dev_dbg(dev, "scheduling asynchronous probe\n"); 983 get_device(dev); 984 async_schedule_dev(__device_attach_async_helper, dev); 985 } else { 986 pm_request_idle(dev); 987 } 988 989 if (dev->parent) 990 pm_runtime_put(dev->parent); 991 } 992 out_unlock: 993 device_unlock(dev); 994 return ret; 995 } 996 997 /** 998 * device_attach - try to attach device to a driver. 999 * @dev: device. 1000 * 1001 * Walk the list of drivers that the bus has and call 1002 * driver_probe_device() for each pair. If a compatible 1003 * pair is found, break out and return. 1004 * 1005 * Returns 1 if the device was bound to a driver; 1006 * 0 if no matching driver was found; 1007 * -ENODEV if the device is not registered. 1008 * 1009 * When called for a USB interface, @dev->parent lock must be held. 1010 */ 1011 int device_attach(struct device *dev) 1012 { 1013 return __device_attach(dev, false); 1014 } 1015 EXPORT_SYMBOL_GPL(device_attach); 1016 1017 void device_initial_probe(struct device *dev) 1018 { 1019 __device_attach(dev, true); 1020 } 1021 1022 /* 1023 * __device_driver_lock - acquire locks needed to manipulate dev->drv 1024 * @dev: Device we will update driver info for 1025 * @parent: Parent device. Needed if the bus requires parent lock 1026 * 1027 * This function will take the required locks for manipulating dev->drv. 1028 * Normally this will just be the @dev lock, but when called for a USB 1029 * interface, @parent lock will be held as well. 1030 */ 1031 static void __device_driver_lock(struct device *dev, struct device *parent) 1032 { 1033 if (parent && dev->bus->need_parent_lock) 1034 device_lock(parent); 1035 device_lock(dev); 1036 } 1037 1038 /* 1039 * __device_driver_unlock - release locks needed to manipulate dev->drv 1040 * @dev: Device we will update driver info for 1041 * @parent: Parent device. Needed if the bus requires parent lock 1042 * 1043 * This function will release the required locks for manipulating dev->drv. 1044 * Normally this will just be the @dev lock, but when called for a 1045 * USB interface, @parent lock will be released as well. 1046 */ 1047 static void __device_driver_unlock(struct device *dev, struct device *parent) 1048 { 1049 device_unlock(dev); 1050 if (parent && dev->bus->need_parent_lock) 1051 device_unlock(parent); 1052 } 1053 1054 /** 1055 * device_driver_attach - attach a specific driver to a specific device 1056 * @drv: Driver to attach 1057 * @dev: Device to attach it to 1058 * 1059 * Manually attach driver to a device. Will acquire both @dev lock and 1060 * @dev->parent lock if needed. Returns 0 on success, -ERR on failure. 1061 */ 1062 int device_driver_attach(struct device_driver *drv, struct device *dev) 1063 { 1064 int ret; 1065 1066 __device_driver_lock(dev, dev->parent); 1067 ret = __driver_probe_device(drv, dev); 1068 __device_driver_unlock(dev, dev->parent); 1069 1070 /* also return probe errors as normal negative errnos */ 1071 if (ret > 0) 1072 ret = -ret; 1073 if (ret == -EPROBE_DEFER) 1074 return -EAGAIN; 1075 return ret; 1076 } 1077 EXPORT_SYMBOL_GPL(device_driver_attach); 1078 1079 static void __driver_attach_async_helper(void *_dev, async_cookie_t cookie) 1080 { 1081 struct device *dev = _dev; 1082 struct device_driver *drv; 1083 int ret; 1084 1085 __device_driver_lock(dev, dev->parent); 1086 drv = dev->p->async_driver; 1087 ret = driver_probe_device(drv, dev); 1088 __device_driver_unlock(dev, dev->parent); 1089 1090 dev_dbg(dev, "driver %s async attach completed: %d\n", drv->name, ret); 1091 1092 put_device(dev); 1093 } 1094 1095 static int __driver_attach(struct device *dev, void *data) 1096 { 1097 struct device_driver *drv = data; 1098 int ret; 1099 1100 /* 1101 * Lock device and try to bind to it. We drop the error 1102 * here and always return 0, because we need to keep trying 1103 * to bind to devices and some drivers will return an error 1104 * simply if it didn't support the device. 1105 * 1106 * driver_probe_device() will spit a warning if there 1107 * is an error. 1108 */ 1109 1110 ret = driver_match_device(drv, dev); 1111 if (ret == 0) { 1112 /* no match */ 1113 return 0; 1114 } else if (ret == -EPROBE_DEFER) { 1115 dev_dbg(dev, "Device match requests probe deferral\n"); 1116 dev->can_match = true; 1117 driver_deferred_probe_add(dev); 1118 } else if (ret < 0) { 1119 dev_dbg(dev, "Bus failed to match device: %d\n", ret); 1120 return ret; 1121 } /* ret > 0 means positive match */ 1122 1123 if (driver_allows_async_probing(drv)) { 1124 /* 1125 * Instead of probing the device synchronously we will 1126 * probe it asynchronously to allow for more parallelism. 1127 * 1128 * We only take the device lock here in order to guarantee 1129 * that the dev->driver and async_driver fields are protected 1130 */ 1131 dev_dbg(dev, "probing driver %s asynchronously\n", drv->name); 1132 device_lock(dev); 1133 if (!dev->driver) { 1134 get_device(dev); 1135 dev->p->async_driver = drv; 1136 async_schedule_dev(__driver_attach_async_helper, dev); 1137 } 1138 device_unlock(dev); 1139 return 0; 1140 } 1141 1142 __device_driver_lock(dev, dev->parent); 1143 driver_probe_device(drv, dev); 1144 __device_driver_unlock(dev, dev->parent); 1145 1146 return 0; 1147 } 1148 1149 /** 1150 * driver_attach - try to bind driver to devices. 1151 * @drv: driver. 1152 * 1153 * Walk the list of devices that the bus has on it and try to 1154 * match the driver with each one. If driver_probe_device() 1155 * returns 0 and the @dev->driver is set, we've found a 1156 * compatible pair. 1157 */ 1158 int driver_attach(struct device_driver *drv) 1159 { 1160 return bus_for_each_dev(drv->bus, NULL, drv, __driver_attach); 1161 } 1162 EXPORT_SYMBOL_GPL(driver_attach); 1163 1164 /* 1165 * __device_release_driver() must be called with @dev lock held. 1166 * When called for a USB interface, @dev->parent lock must be held as well. 1167 */ 1168 static void __device_release_driver(struct device *dev, struct device *parent) 1169 { 1170 struct device_driver *drv; 1171 1172 drv = dev->driver; 1173 if (drv) { 1174 pm_runtime_get_sync(dev); 1175 1176 while (device_links_busy(dev)) { 1177 __device_driver_unlock(dev, parent); 1178 1179 device_links_unbind_consumers(dev); 1180 1181 __device_driver_lock(dev, parent); 1182 /* 1183 * A concurrent invocation of the same function might 1184 * have released the driver successfully while this one 1185 * was waiting, so check for that. 1186 */ 1187 if (dev->driver != drv) { 1188 pm_runtime_put(dev); 1189 return; 1190 } 1191 } 1192 1193 driver_sysfs_remove(dev); 1194 1195 if (dev->bus) 1196 blocking_notifier_call_chain(&dev->bus->p->bus_notifier, 1197 BUS_NOTIFY_UNBIND_DRIVER, 1198 dev); 1199 1200 pm_runtime_put_sync(dev); 1201 1202 device_remove(dev); 1203 1204 if (dev->bus && dev->bus->dma_cleanup) 1205 dev->bus->dma_cleanup(dev); 1206 1207 device_links_driver_cleanup(dev); 1208 device_unbind_cleanup(dev); 1209 1210 klist_remove(&dev->p->knode_driver); 1211 device_pm_check_callbacks(dev); 1212 if (dev->bus) 1213 blocking_notifier_call_chain(&dev->bus->p->bus_notifier, 1214 BUS_NOTIFY_UNBOUND_DRIVER, 1215 dev); 1216 1217 kobject_uevent(&dev->kobj, KOBJ_UNBIND); 1218 } 1219 } 1220 1221 void device_release_driver_internal(struct device *dev, 1222 struct device_driver *drv, 1223 struct device *parent) 1224 { 1225 __device_driver_lock(dev, parent); 1226 1227 if (!drv || drv == dev->driver) 1228 __device_release_driver(dev, parent); 1229 1230 __device_driver_unlock(dev, parent); 1231 } 1232 1233 /** 1234 * device_release_driver - manually detach device from driver. 1235 * @dev: device. 1236 * 1237 * Manually detach device from driver. 1238 * When called for a USB interface, @dev->parent lock must be held. 1239 * 1240 * If this function is to be called with @dev->parent lock held, ensure that 1241 * the device's consumers are unbound in advance or that their locks can be 1242 * acquired under the @dev->parent lock. 1243 */ 1244 void device_release_driver(struct device *dev) 1245 { 1246 /* 1247 * If anyone calls device_release_driver() recursively from 1248 * within their ->remove callback for the same device, they 1249 * will deadlock right here. 1250 */ 1251 device_release_driver_internal(dev, NULL, NULL); 1252 } 1253 EXPORT_SYMBOL_GPL(device_release_driver); 1254 1255 /** 1256 * device_driver_detach - detach driver from a specific device 1257 * @dev: device to detach driver from 1258 * 1259 * Detach driver from device. Will acquire both @dev lock and @dev->parent 1260 * lock if needed. 1261 */ 1262 void device_driver_detach(struct device *dev) 1263 { 1264 device_release_driver_internal(dev, NULL, dev->parent); 1265 } 1266 1267 /** 1268 * driver_detach - detach driver from all devices it controls. 1269 * @drv: driver. 1270 */ 1271 void driver_detach(struct device_driver *drv) 1272 { 1273 struct device_private *dev_prv; 1274 struct device *dev; 1275 1276 if (driver_allows_async_probing(drv)) 1277 async_synchronize_full(); 1278 1279 for (;;) { 1280 spin_lock(&drv->p->klist_devices.k_lock); 1281 if (list_empty(&drv->p->klist_devices.k_list)) { 1282 spin_unlock(&drv->p->klist_devices.k_lock); 1283 break; 1284 } 1285 dev_prv = list_last_entry(&drv->p->klist_devices.k_list, 1286 struct device_private, 1287 knode_driver.n_node); 1288 dev = dev_prv->device; 1289 get_device(dev); 1290 spin_unlock(&drv->p->klist_devices.k_lock); 1291 device_release_driver_internal(dev, drv, dev->parent); 1292 put_device(dev); 1293 } 1294 } 1295