xref: /openbmc/linux/block/genhd.c (revision 02d89917)
1  // SPDX-License-Identifier: GPL-2.0
2  /*
3   *  gendisk handling
4   *
5   * Portions Copyright (C) 2020 Christoph Hellwig
6   */
7  
8  #include <linux/module.h>
9  #include <linux/ctype.h>
10  #include <linux/fs.h>
11  #include <linux/kdev_t.h>
12  #include <linux/kernel.h>
13  #include <linux/blkdev.h>
14  #include <linux/backing-dev.h>
15  #include <linux/init.h>
16  #include <linux/spinlock.h>
17  #include <linux/proc_fs.h>
18  #include <linux/seq_file.h>
19  #include <linux/slab.h>
20  #include <linux/kmod.h>
21  #include <linux/major.h>
22  #include <linux/mutex.h>
23  #include <linux/idr.h>
24  #include <linux/log2.h>
25  #include <linux/pm_runtime.h>
26  #include <linux/badblocks.h>
27  #include <linux/part_stat.h>
28  #include <linux/blktrace_api.h>
29  
30  #include "blk-throttle.h"
31  #include "blk.h"
32  #include "blk-mq-sched.h"
33  #include "blk-rq-qos.h"
34  #include "blk-cgroup.h"
35  
36  static struct kobject *block_depr;
37  
38  /*
39   * Unique, monotonically increasing sequential number associated with block
40   * devices instances (i.e. incremented each time a device is attached).
41   * Associating uevents with block devices in userspace is difficult and racy:
42   * the uevent netlink socket is lossy, and on slow and overloaded systems has
43   * a very high latency.
44   * Block devices do not have exclusive owners in userspace, any process can set
45   * one up (e.g. loop devices). Moreover, device names can be reused (e.g. loop0
46   * can be reused again and again).
47   * A userspace process setting up a block device and watching for its events
48   * cannot thus reliably tell whether an event relates to the device it just set
49   * up or another earlier instance with the same name.
50   * This sequential number allows userspace processes to solve this problem, and
51   * uniquely associate an uevent to the lifetime to a device.
52   */
53  static atomic64_t diskseq;
54  
55  /* for extended dynamic devt allocation, currently only one major is used */
56  #define NR_EXT_DEVT		(1 << MINORBITS)
57  static DEFINE_IDA(ext_devt_ida);
58  
59  void set_capacity(struct gendisk *disk, sector_t sectors)
60  {
61  	bdev_set_nr_sectors(disk->part0, sectors);
62  }
63  EXPORT_SYMBOL(set_capacity);
64  
65  /*
66   * Set disk capacity and notify if the size is not currently zero and will not
67   * be set to zero.  Returns true if a uevent was sent, otherwise false.
68   */
69  bool set_capacity_and_notify(struct gendisk *disk, sector_t size)
70  {
71  	sector_t capacity = get_capacity(disk);
72  	char *envp[] = { "RESIZE=1", NULL };
73  
74  	set_capacity(disk, size);
75  
76  	/*
77  	 * Only print a message and send a uevent if the gendisk is user visible
78  	 * and alive.  This avoids spamming the log and udev when setting the
79  	 * initial capacity during probing.
80  	 */
81  	if (size == capacity ||
82  	    !disk_live(disk) ||
83  	    (disk->flags & GENHD_FL_HIDDEN))
84  		return false;
85  
86  	pr_info("%s: detected capacity change from %lld to %lld\n",
87  		disk->disk_name, capacity, size);
88  
89  	/*
90  	 * Historically we did not send a uevent for changes to/from an empty
91  	 * device.
92  	 */
93  	if (!capacity || !size)
94  		return false;
95  	kobject_uevent_env(&disk_to_dev(disk)->kobj, KOBJ_CHANGE, envp);
96  	return true;
97  }
98  EXPORT_SYMBOL_GPL(set_capacity_and_notify);
99  
100  static void part_stat_read_all(struct block_device *part,
101  		struct disk_stats *stat)
102  {
103  	int cpu;
104  
105  	memset(stat, 0, sizeof(struct disk_stats));
106  	for_each_possible_cpu(cpu) {
107  		struct disk_stats *ptr = per_cpu_ptr(part->bd_stats, cpu);
108  		int group;
109  
110  		for (group = 0; group < NR_STAT_GROUPS; group++) {
111  			stat->nsecs[group] += ptr->nsecs[group];
112  			stat->sectors[group] += ptr->sectors[group];
113  			stat->ios[group] += ptr->ios[group];
114  			stat->merges[group] += ptr->merges[group];
115  		}
116  
117  		stat->io_ticks += ptr->io_ticks;
118  	}
119  }
120  
121  static unsigned int part_in_flight(struct block_device *part)
122  {
123  	unsigned int inflight = 0;
124  	int cpu;
125  
126  	for_each_possible_cpu(cpu) {
127  		inflight += part_stat_local_read_cpu(part, in_flight[0], cpu) +
128  			    part_stat_local_read_cpu(part, in_flight[1], cpu);
129  	}
130  	if ((int)inflight < 0)
131  		inflight = 0;
132  
133  	return inflight;
134  }
135  
136  static void part_in_flight_rw(struct block_device *part,
137  		unsigned int inflight[2])
138  {
139  	int cpu;
140  
141  	inflight[0] = 0;
142  	inflight[1] = 0;
143  	for_each_possible_cpu(cpu) {
144  		inflight[0] += part_stat_local_read_cpu(part, in_flight[0], cpu);
145  		inflight[1] += part_stat_local_read_cpu(part, in_flight[1], cpu);
146  	}
147  	if ((int)inflight[0] < 0)
148  		inflight[0] = 0;
149  	if ((int)inflight[1] < 0)
150  		inflight[1] = 0;
151  }
152  
153  /*
154   * Can be deleted altogether. Later.
155   *
156   */
157  #define BLKDEV_MAJOR_HASH_SIZE 255
158  static struct blk_major_name {
159  	struct blk_major_name *next;
160  	int major;
161  	char name[16];
162  #ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
163  	void (*probe)(dev_t devt);
164  #endif
165  } *major_names[BLKDEV_MAJOR_HASH_SIZE];
166  static DEFINE_MUTEX(major_names_lock);
167  static DEFINE_SPINLOCK(major_names_spinlock);
168  
169  /* index in the above - for now: assume no multimajor ranges */
170  static inline int major_to_index(unsigned major)
171  {
172  	return major % BLKDEV_MAJOR_HASH_SIZE;
173  }
174  
175  #ifdef CONFIG_PROC_FS
176  void blkdev_show(struct seq_file *seqf, off_t offset)
177  {
178  	struct blk_major_name *dp;
179  
180  	spin_lock(&major_names_spinlock);
181  	for (dp = major_names[major_to_index(offset)]; dp; dp = dp->next)
182  		if (dp->major == offset)
183  			seq_printf(seqf, "%3d %s\n", dp->major, dp->name);
184  	spin_unlock(&major_names_spinlock);
185  }
186  #endif /* CONFIG_PROC_FS */
187  
188  /**
189   * __register_blkdev - register a new block device
190   *
191   * @major: the requested major device number [1..BLKDEV_MAJOR_MAX-1]. If
192   *         @major = 0, try to allocate any unused major number.
193   * @name: the name of the new block device as a zero terminated string
194   * @probe: pre-devtmpfs / pre-udev callback used to create disks when their
195   *	   pre-created device node is accessed. When a probe call uses
196   *	   add_disk() and it fails the driver must cleanup resources. This
197   *	   interface may soon be removed.
198   *
199   * The @name must be unique within the system.
200   *
201   * The return value depends on the @major input parameter:
202   *
203   *  - if a major device number was requested in range [1..BLKDEV_MAJOR_MAX-1]
204   *    then the function returns zero on success, or a negative error code
205   *  - if any unused major number was requested with @major = 0 parameter
206   *    then the return value is the allocated major number in range
207   *    [1..BLKDEV_MAJOR_MAX-1] or a negative error code otherwise
208   *
209   * See Documentation/admin-guide/devices.txt for the list of allocated
210   * major numbers.
211   *
212   * Use register_blkdev instead for any new code.
213   */
214  int __register_blkdev(unsigned int major, const char *name,
215  		void (*probe)(dev_t devt))
216  {
217  	struct blk_major_name **n, *p;
218  	int index, ret = 0;
219  
220  	mutex_lock(&major_names_lock);
221  
222  	/* temporary */
223  	if (major == 0) {
224  		for (index = ARRAY_SIZE(major_names)-1; index > 0; index--) {
225  			if (major_names[index] == NULL)
226  				break;
227  		}
228  
229  		if (index == 0) {
230  			printk("%s: failed to get major for %s\n",
231  			       __func__, name);
232  			ret = -EBUSY;
233  			goto out;
234  		}
235  		major = index;
236  		ret = major;
237  	}
238  
239  	if (major >= BLKDEV_MAJOR_MAX) {
240  		pr_err("%s: major requested (%u) is greater than the maximum (%u) for %s\n",
241  		       __func__, major, BLKDEV_MAJOR_MAX-1, name);
242  
243  		ret = -EINVAL;
244  		goto out;
245  	}
246  
247  	p = kmalloc(sizeof(struct blk_major_name), GFP_KERNEL);
248  	if (p == NULL) {
249  		ret = -ENOMEM;
250  		goto out;
251  	}
252  
253  	p->major = major;
254  #ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
255  	p->probe = probe;
256  #endif
257  	strscpy(p->name, name, sizeof(p->name));
258  	p->next = NULL;
259  	index = major_to_index(major);
260  
261  	spin_lock(&major_names_spinlock);
262  	for (n = &major_names[index]; *n; n = &(*n)->next) {
263  		if ((*n)->major == major)
264  			break;
265  	}
266  	if (!*n)
267  		*n = p;
268  	else
269  		ret = -EBUSY;
270  	spin_unlock(&major_names_spinlock);
271  
272  	if (ret < 0) {
273  		printk("register_blkdev: cannot get major %u for %s\n",
274  		       major, name);
275  		kfree(p);
276  	}
277  out:
278  	mutex_unlock(&major_names_lock);
279  	return ret;
280  }
281  EXPORT_SYMBOL(__register_blkdev);
282  
283  void unregister_blkdev(unsigned int major, const char *name)
284  {
285  	struct blk_major_name **n;
286  	struct blk_major_name *p = NULL;
287  	int index = major_to_index(major);
288  
289  	mutex_lock(&major_names_lock);
290  	spin_lock(&major_names_spinlock);
291  	for (n = &major_names[index]; *n; n = &(*n)->next)
292  		if ((*n)->major == major)
293  			break;
294  	if (!*n || strcmp((*n)->name, name)) {
295  		WARN_ON(1);
296  	} else {
297  		p = *n;
298  		*n = p->next;
299  	}
300  	spin_unlock(&major_names_spinlock);
301  	mutex_unlock(&major_names_lock);
302  	kfree(p);
303  }
304  
305  EXPORT_SYMBOL(unregister_blkdev);
306  
307  int blk_alloc_ext_minor(void)
308  {
309  	int idx;
310  
311  	idx = ida_alloc_range(&ext_devt_ida, 0, NR_EXT_DEVT - 1, GFP_KERNEL);
312  	if (idx == -ENOSPC)
313  		return -EBUSY;
314  	return idx;
315  }
316  
317  void blk_free_ext_minor(unsigned int minor)
318  {
319  	ida_free(&ext_devt_ida, minor);
320  }
321  
322  void disk_uevent(struct gendisk *disk, enum kobject_action action)
323  {
324  	struct block_device *part;
325  	unsigned long idx;
326  
327  	rcu_read_lock();
328  	xa_for_each(&disk->part_tbl, idx, part) {
329  		if (bdev_is_partition(part) && !bdev_nr_sectors(part))
330  			continue;
331  		if (!kobject_get_unless_zero(&part->bd_device.kobj))
332  			continue;
333  
334  		rcu_read_unlock();
335  		kobject_uevent(bdev_kobj(part), action);
336  		put_device(&part->bd_device);
337  		rcu_read_lock();
338  	}
339  	rcu_read_unlock();
340  }
341  EXPORT_SYMBOL_GPL(disk_uevent);
342  
343  int disk_scan_partitions(struct gendisk *disk, blk_mode_t mode)
344  {
345  	struct block_device *bdev;
346  	int ret = 0;
347  
348  	if (disk->flags & (GENHD_FL_NO_PART | GENHD_FL_HIDDEN))
349  		return -EINVAL;
350  	if (test_bit(GD_SUPPRESS_PART_SCAN, &disk->state))
351  		return -EINVAL;
352  	if (disk->open_partitions)
353  		return -EBUSY;
354  
355  	/*
356  	 * If the device is opened exclusively by current thread already, it's
357  	 * safe to scan partitons, otherwise, use bd_prepare_to_claim() to
358  	 * synchronize with other exclusive openers and other partition
359  	 * scanners.
360  	 */
361  	if (!(mode & BLK_OPEN_EXCL)) {
362  		ret = bd_prepare_to_claim(disk->part0, disk_scan_partitions,
363  					  NULL);
364  		if (ret)
365  			return ret;
366  	}
367  
368  	set_bit(GD_NEED_PART_SCAN, &disk->state);
369  	bdev = blkdev_get_by_dev(disk_devt(disk), mode & ~BLK_OPEN_EXCL, NULL,
370  				 NULL);
371  	if (IS_ERR(bdev))
372  		ret =  PTR_ERR(bdev);
373  	else
374  		blkdev_put(bdev, NULL);
375  
376  	/*
377  	 * If blkdev_get_by_dev() failed early, GD_NEED_PART_SCAN is still set,
378  	 * and this will cause that re-assemble partitioned raid device will
379  	 * creat partition for underlying disk.
380  	 */
381  	clear_bit(GD_NEED_PART_SCAN, &disk->state);
382  	if (!(mode & BLK_OPEN_EXCL))
383  		bd_abort_claiming(disk->part0, disk_scan_partitions);
384  	return ret;
385  }
386  
387  /**
388   * device_add_disk - add disk information to kernel list
389   * @parent: parent device for the disk
390   * @disk: per-device partitioning information
391   * @groups: Additional per-device sysfs groups
392   *
393   * This function registers the partitioning information in @disk
394   * with the kernel.
395   */
396  int __must_check device_add_disk(struct device *parent, struct gendisk *disk,
397  				 const struct attribute_group **groups)
398  
399  {
400  	struct device *ddev = disk_to_dev(disk);
401  	int ret;
402  
403  	/* Only makes sense for bio-based to set ->poll_bio */
404  	if (queue_is_mq(disk->queue) && disk->fops->poll_bio)
405  		return -EINVAL;
406  
407  	/*
408  	 * The disk queue should now be all set with enough information about
409  	 * the device for the elevator code to pick an adequate default
410  	 * elevator if one is needed, that is, for devices requesting queue
411  	 * registration.
412  	 */
413  	elevator_init_mq(disk->queue);
414  
415  	/* Mark bdev as having a submit_bio, if needed */
416  	disk->part0->bd_has_submit_bio = disk->fops->submit_bio != NULL;
417  
418  	/*
419  	 * If the driver provides an explicit major number it also must provide
420  	 * the number of minors numbers supported, and those will be used to
421  	 * setup the gendisk.
422  	 * Otherwise just allocate the device numbers for both the whole device
423  	 * and all partitions from the extended dev_t space.
424  	 */
425  	ret = -EINVAL;
426  	if (disk->major) {
427  		if (WARN_ON(!disk->minors))
428  			goto out_exit_elevator;
429  
430  		if (disk->minors > DISK_MAX_PARTS) {
431  			pr_err("block: can't allocate more than %d partitions\n",
432  				DISK_MAX_PARTS);
433  			disk->minors = DISK_MAX_PARTS;
434  		}
435  		if (disk->first_minor + disk->minors > MINORMASK + 1)
436  			goto out_exit_elevator;
437  	} else {
438  		if (WARN_ON(disk->minors))
439  			goto out_exit_elevator;
440  
441  		ret = blk_alloc_ext_minor();
442  		if (ret < 0)
443  			goto out_exit_elevator;
444  		disk->major = BLOCK_EXT_MAJOR;
445  		disk->first_minor = ret;
446  	}
447  
448  	/* delay uevents, until we scanned partition table */
449  	dev_set_uevent_suppress(ddev, 1);
450  
451  	ddev->parent = parent;
452  	ddev->groups = groups;
453  	dev_set_name(ddev, "%s", disk->disk_name);
454  	if (!(disk->flags & GENHD_FL_HIDDEN))
455  		ddev->devt = MKDEV(disk->major, disk->first_minor);
456  	ret = device_add(ddev);
457  	if (ret)
458  		goto out_free_ext_minor;
459  
460  	ret = disk_alloc_events(disk);
461  	if (ret)
462  		goto out_device_del;
463  
464  	ret = sysfs_create_link(block_depr, &ddev->kobj,
465  				kobject_name(&ddev->kobj));
466  	if (ret)
467  		goto out_device_del;
468  
469  	/*
470  	 * avoid probable deadlock caused by allocating memory with
471  	 * GFP_KERNEL in runtime_resume callback of its all ancestor
472  	 * devices
473  	 */
474  	pm_runtime_set_memalloc_noio(ddev, true);
475  
476  	disk->part0->bd_holder_dir =
477  		kobject_create_and_add("holders", &ddev->kobj);
478  	if (!disk->part0->bd_holder_dir) {
479  		ret = -ENOMEM;
480  		goto out_del_block_link;
481  	}
482  	disk->slave_dir = kobject_create_and_add("slaves", &ddev->kobj);
483  	if (!disk->slave_dir) {
484  		ret = -ENOMEM;
485  		goto out_put_holder_dir;
486  	}
487  
488  	ret = blk_register_queue(disk);
489  	if (ret)
490  		goto out_put_slave_dir;
491  
492  	if (!(disk->flags & GENHD_FL_HIDDEN)) {
493  		ret = bdi_register(disk->bdi, "%u:%u",
494  				   disk->major, disk->first_minor);
495  		if (ret)
496  			goto out_unregister_queue;
497  		bdi_set_owner(disk->bdi, ddev);
498  		ret = sysfs_create_link(&ddev->kobj,
499  					&disk->bdi->dev->kobj, "bdi");
500  		if (ret)
501  			goto out_unregister_bdi;
502  
503  		/* Make sure the first partition scan will be proceed */
504  		if (get_capacity(disk) && !(disk->flags & GENHD_FL_NO_PART) &&
505  		    !test_bit(GD_SUPPRESS_PART_SCAN, &disk->state))
506  			set_bit(GD_NEED_PART_SCAN, &disk->state);
507  
508  		bdev_add(disk->part0, ddev->devt);
509  		if (get_capacity(disk))
510  			disk_scan_partitions(disk, BLK_OPEN_READ);
511  
512  		/*
513  		 * Announce the disk and partitions after all partitions are
514  		 * created. (for hidden disks uevents remain suppressed forever)
515  		 */
516  		dev_set_uevent_suppress(ddev, 0);
517  		disk_uevent(disk, KOBJ_ADD);
518  	} else {
519  		/*
520  		 * Even if the block_device for a hidden gendisk is not
521  		 * registered, it needs to have a valid bd_dev so that the
522  		 * freeing of the dynamic major works.
523  		 */
524  		disk->part0->bd_dev = MKDEV(disk->major, disk->first_minor);
525  	}
526  
527  	disk_update_readahead(disk);
528  	disk_add_events(disk);
529  	set_bit(GD_ADDED, &disk->state);
530  	return 0;
531  
532  out_unregister_bdi:
533  	if (!(disk->flags & GENHD_FL_HIDDEN))
534  		bdi_unregister(disk->bdi);
535  out_unregister_queue:
536  	blk_unregister_queue(disk);
537  	rq_qos_exit(disk->queue);
538  out_put_slave_dir:
539  	kobject_put(disk->slave_dir);
540  	disk->slave_dir = NULL;
541  out_put_holder_dir:
542  	kobject_put(disk->part0->bd_holder_dir);
543  out_del_block_link:
544  	sysfs_remove_link(block_depr, dev_name(ddev));
545  out_device_del:
546  	device_del(ddev);
547  out_free_ext_minor:
548  	if (disk->major == BLOCK_EXT_MAJOR)
549  		blk_free_ext_minor(disk->first_minor);
550  out_exit_elevator:
551  	if (disk->queue->elevator)
552  		elevator_exit(disk->queue);
553  	return ret;
554  }
555  EXPORT_SYMBOL(device_add_disk);
556  
557  static void blk_report_disk_dead(struct gendisk *disk, bool surprise)
558  {
559  	struct block_device *bdev;
560  	unsigned long idx;
561  
562  	rcu_read_lock();
563  	xa_for_each(&disk->part_tbl, idx, bdev) {
564  		if (!kobject_get_unless_zero(&bdev->bd_device.kobj))
565  			continue;
566  		rcu_read_unlock();
567  
568  		bdev_mark_dead(bdev, surprise);
569  
570  		put_device(&bdev->bd_device);
571  		rcu_read_lock();
572  	}
573  	rcu_read_unlock();
574  }
575  
576  static void __blk_mark_disk_dead(struct gendisk *disk)
577  {
578  	/*
579  	 * Fail any new I/O.
580  	 */
581  	if (test_and_set_bit(GD_DEAD, &disk->state))
582  		return;
583  
584  	if (test_bit(GD_OWNS_QUEUE, &disk->state))
585  		blk_queue_flag_set(QUEUE_FLAG_DYING, disk->queue);
586  
587  	/*
588  	 * Stop buffered writers from dirtying pages that can't be written out.
589  	 */
590  	set_capacity(disk, 0);
591  
592  	/*
593  	 * Prevent new I/O from crossing bio_queue_enter().
594  	 */
595  	blk_queue_start_drain(disk->queue);
596  }
597  
598  /**
599   * blk_mark_disk_dead - mark a disk as dead
600   * @disk: disk to mark as dead
601   *
602   * Mark as disk as dead (e.g. surprise removed) and don't accept any new I/O
603   * to this disk.
604   */
605  void blk_mark_disk_dead(struct gendisk *disk)
606  {
607  	__blk_mark_disk_dead(disk);
608  	blk_report_disk_dead(disk, true);
609  }
610  EXPORT_SYMBOL_GPL(blk_mark_disk_dead);
611  
612  /**
613   * del_gendisk - remove the gendisk
614   * @disk: the struct gendisk to remove
615   *
616   * Removes the gendisk and all its associated resources. This deletes the
617   * partitions associated with the gendisk, and unregisters the associated
618   * request_queue.
619   *
620   * This is the counter to the respective __device_add_disk() call.
621   *
622   * The final removal of the struct gendisk happens when its refcount reaches 0
623   * with put_disk(), which should be called after del_gendisk(), if
624   * __device_add_disk() was used.
625   *
626   * Drivers exist which depend on the release of the gendisk to be synchronous,
627   * it should not be deferred.
628   *
629   * Context: can sleep
630   */
631  void del_gendisk(struct gendisk *disk)
632  {
633  	struct request_queue *q = disk->queue;
634  	struct block_device *part;
635  	unsigned long idx;
636  
637  	might_sleep();
638  
639  	if (WARN_ON_ONCE(!disk_live(disk) && !(disk->flags & GENHD_FL_HIDDEN)))
640  		return;
641  
642  	disk_del_events(disk);
643  
644  	/*
645  	 * Prevent new openers by unlinked the bdev inode.
646  	 */
647  	mutex_lock(&disk->open_mutex);
648  	xa_for_each(&disk->part_tbl, idx, part)
649  		remove_inode_hash(part->bd_inode);
650  	mutex_unlock(&disk->open_mutex);
651  
652  	/*
653  	 * Tell the file system to write back all dirty data and shut down if
654  	 * it hasn't been notified earlier.
655  	 */
656  	if (!test_bit(GD_DEAD, &disk->state))
657  		blk_report_disk_dead(disk, false);
658  	__blk_mark_disk_dead(disk);
659  
660  	/*
661  	 * Drop all partitions now that the disk is marked dead.
662  	 */
663  	mutex_lock(&disk->open_mutex);
664  	xa_for_each_start(&disk->part_tbl, idx, part, 1)
665  		drop_partition(part);
666  	mutex_unlock(&disk->open_mutex);
667  
668  	if (!(disk->flags & GENHD_FL_HIDDEN)) {
669  		sysfs_remove_link(&disk_to_dev(disk)->kobj, "bdi");
670  
671  		/*
672  		 * Unregister bdi before releasing device numbers (as they can
673  		 * get reused and we'd get clashes in sysfs).
674  		 */
675  		bdi_unregister(disk->bdi);
676  	}
677  
678  	blk_unregister_queue(disk);
679  
680  	kobject_put(disk->part0->bd_holder_dir);
681  	kobject_put(disk->slave_dir);
682  	disk->slave_dir = NULL;
683  
684  	part_stat_set_all(disk->part0, 0);
685  	disk->part0->bd_stamp = 0;
686  	sysfs_remove_link(block_depr, dev_name(disk_to_dev(disk)));
687  	pm_runtime_set_memalloc_noio(disk_to_dev(disk), false);
688  	device_del(disk_to_dev(disk));
689  
690  	blk_mq_freeze_queue_wait(q);
691  
692  	blk_throtl_cancel_bios(disk);
693  
694  	blk_sync_queue(q);
695  	blk_flush_integrity();
696  
697  	if (queue_is_mq(q))
698  		blk_mq_cancel_work_sync(q);
699  
700  	blk_mq_quiesce_queue(q);
701  	if (q->elevator) {
702  		mutex_lock(&q->sysfs_lock);
703  		elevator_exit(q);
704  		mutex_unlock(&q->sysfs_lock);
705  	}
706  	rq_qos_exit(q);
707  	blk_mq_unquiesce_queue(q);
708  
709  	/*
710  	 * If the disk does not own the queue, allow using passthrough requests
711  	 * again.  Else leave the queue frozen to fail all I/O.
712  	 */
713  	if (!test_bit(GD_OWNS_QUEUE, &disk->state)) {
714  		blk_queue_flag_clear(QUEUE_FLAG_INIT_DONE, q);
715  		__blk_mq_unfreeze_queue(q, true);
716  	} else {
717  		if (queue_is_mq(q))
718  			blk_mq_exit_queue(q);
719  	}
720  }
721  EXPORT_SYMBOL(del_gendisk);
722  
723  /**
724   * invalidate_disk - invalidate the disk
725   * @disk: the struct gendisk to invalidate
726   *
727   * A helper to invalidates the disk. It will clean the disk's associated
728   * buffer/page caches and reset its internal states so that the disk
729   * can be reused by the drivers.
730   *
731   * Context: can sleep
732   */
733  void invalidate_disk(struct gendisk *disk)
734  {
735  	struct block_device *bdev = disk->part0;
736  
737  	invalidate_bdev(bdev);
738  	bdev->bd_inode->i_mapping->wb_err = 0;
739  	set_capacity(disk, 0);
740  }
741  EXPORT_SYMBOL(invalidate_disk);
742  
743  /* sysfs access to bad-blocks list. */
744  static ssize_t disk_badblocks_show(struct device *dev,
745  					struct device_attribute *attr,
746  					char *page)
747  {
748  	struct gendisk *disk = dev_to_disk(dev);
749  
750  	if (!disk->bb)
751  		return sprintf(page, "\n");
752  
753  	return badblocks_show(disk->bb, page, 0);
754  }
755  
756  static ssize_t disk_badblocks_store(struct device *dev,
757  					struct device_attribute *attr,
758  					const char *page, size_t len)
759  {
760  	struct gendisk *disk = dev_to_disk(dev);
761  
762  	if (!disk->bb)
763  		return -ENXIO;
764  
765  	return badblocks_store(disk->bb, page, len, 0);
766  }
767  
768  #ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
769  void blk_request_module(dev_t devt)
770  {
771  	unsigned int major = MAJOR(devt);
772  	struct blk_major_name **n;
773  
774  	mutex_lock(&major_names_lock);
775  	for (n = &major_names[major_to_index(major)]; *n; n = &(*n)->next) {
776  		if ((*n)->major == major && (*n)->probe) {
777  			(*n)->probe(devt);
778  			mutex_unlock(&major_names_lock);
779  			return;
780  		}
781  	}
782  	mutex_unlock(&major_names_lock);
783  
784  	if (request_module("block-major-%d-%d", MAJOR(devt), MINOR(devt)) > 0)
785  		/* Make old-style 2.4 aliases work */
786  		request_module("block-major-%d", MAJOR(devt));
787  }
788  #endif /* CONFIG_BLOCK_LEGACY_AUTOLOAD */
789  
790  #ifdef CONFIG_PROC_FS
791  /* iterator */
792  static void *disk_seqf_start(struct seq_file *seqf, loff_t *pos)
793  {
794  	loff_t skip = *pos;
795  	struct class_dev_iter *iter;
796  	struct device *dev;
797  
798  	iter = kmalloc(sizeof(*iter), GFP_KERNEL);
799  	if (!iter)
800  		return ERR_PTR(-ENOMEM);
801  
802  	seqf->private = iter;
803  	class_dev_iter_init(iter, &block_class, NULL, &disk_type);
804  	do {
805  		dev = class_dev_iter_next(iter);
806  		if (!dev)
807  			return NULL;
808  	} while (skip--);
809  
810  	return dev_to_disk(dev);
811  }
812  
813  static void *disk_seqf_next(struct seq_file *seqf, void *v, loff_t *pos)
814  {
815  	struct device *dev;
816  
817  	(*pos)++;
818  	dev = class_dev_iter_next(seqf->private);
819  	if (dev)
820  		return dev_to_disk(dev);
821  
822  	return NULL;
823  }
824  
825  static void disk_seqf_stop(struct seq_file *seqf, void *v)
826  {
827  	struct class_dev_iter *iter = seqf->private;
828  
829  	/* stop is called even after start failed :-( */
830  	if (iter) {
831  		class_dev_iter_exit(iter);
832  		kfree(iter);
833  		seqf->private = NULL;
834  	}
835  }
836  
837  static void *show_partition_start(struct seq_file *seqf, loff_t *pos)
838  {
839  	void *p;
840  
841  	p = disk_seqf_start(seqf, pos);
842  	if (!IS_ERR_OR_NULL(p) && !*pos)
843  		seq_puts(seqf, "major minor  #blocks  name\n\n");
844  	return p;
845  }
846  
847  static int show_partition(struct seq_file *seqf, void *v)
848  {
849  	struct gendisk *sgp = v;
850  	struct block_device *part;
851  	unsigned long idx;
852  
853  	if (!get_capacity(sgp) || (sgp->flags & GENHD_FL_HIDDEN))
854  		return 0;
855  
856  	rcu_read_lock();
857  	xa_for_each(&sgp->part_tbl, idx, part) {
858  		if (!bdev_nr_sectors(part))
859  			continue;
860  		seq_printf(seqf, "%4d  %7d %10llu %pg\n",
861  			   MAJOR(part->bd_dev), MINOR(part->bd_dev),
862  			   bdev_nr_sectors(part) >> 1, part);
863  	}
864  	rcu_read_unlock();
865  	return 0;
866  }
867  
868  static const struct seq_operations partitions_op = {
869  	.start	= show_partition_start,
870  	.next	= disk_seqf_next,
871  	.stop	= disk_seqf_stop,
872  	.show	= show_partition
873  };
874  #endif
875  
876  static int __init genhd_device_init(void)
877  {
878  	int error;
879  
880  	error = class_register(&block_class);
881  	if (unlikely(error))
882  		return error;
883  	blk_dev_init();
884  
885  	register_blkdev(BLOCK_EXT_MAJOR, "blkext");
886  
887  	/* create top-level block dir */
888  	block_depr = kobject_create_and_add("block", NULL);
889  	return 0;
890  }
891  
892  subsys_initcall(genhd_device_init);
893  
894  static ssize_t disk_range_show(struct device *dev,
895  			       struct device_attribute *attr, char *buf)
896  {
897  	struct gendisk *disk = dev_to_disk(dev);
898  
899  	return sprintf(buf, "%d\n", disk->minors);
900  }
901  
902  static ssize_t disk_ext_range_show(struct device *dev,
903  				   struct device_attribute *attr, char *buf)
904  {
905  	struct gendisk *disk = dev_to_disk(dev);
906  
907  	return sprintf(buf, "%d\n",
908  		(disk->flags & GENHD_FL_NO_PART) ? 1 : DISK_MAX_PARTS);
909  }
910  
911  static ssize_t disk_removable_show(struct device *dev,
912  				   struct device_attribute *attr, char *buf)
913  {
914  	struct gendisk *disk = dev_to_disk(dev);
915  
916  	return sprintf(buf, "%d\n",
917  		       (disk->flags & GENHD_FL_REMOVABLE ? 1 : 0));
918  }
919  
920  static ssize_t disk_hidden_show(struct device *dev,
921  				   struct device_attribute *attr, char *buf)
922  {
923  	struct gendisk *disk = dev_to_disk(dev);
924  
925  	return sprintf(buf, "%d\n",
926  		       (disk->flags & GENHD_FL_HIDDEN ? 1 : 0));
927  }
928  
929  static ssize_t disk_ro_show(struct device *dev,
930  				   struct device_attribute *attr, char *buf)
931  {
932  	struct gendisk *disk = dev_to_disk(dev);
933  
934  	return sprintf(buf, "%d\n", get_disk_ro(disk) ? 1 : 0);
935  }
936  
937  ssize_t part_size_show(struct device *dev,
938  		       struct device_attribute *attr, char *buf)
939  {
940  	return sprintf(buf, "%llu\n", bdev_nr_sectors(dev_to_bdev(dev)));
941  }
942  
943  ssize_t part_stat_show(struct device *dev,
944  		       struct device_attribute *attr, char *buf)
945  {
946  	struct block_device *bdev = dev_to_bdev(dev);
947  	struct request_queue *q = bdev_get_queue(bdev);
948  	struct disk_stats stat;
949  	unsigned int inflight;
950  
951  	if (queue_is_mq(q))
952  		inflight = blk_mq_in_flight(q, bdev);
953  	else
954  		inflight = part_in_flight(bdev);
955  
956  	if (inflight) {
957  		part_stat_lock();
958  		update_io_ticks(bdev, jiffies, true);
959  		part_stat_unlock();
960  	}
961  	part_stat_read_all(bdev, &stat);
962  	return sprintf(buf,
963  		"%8lu %8lu %8llu %8u "
964  		"%8lu %8lu %8llu %8u "
965  		"%8u %8u %8u "
966  		"%8lu %8lu %8llu %8u "
967  		"%8lu %8u"
968  		"\n",
969  		stat.ios[STAT_READ],
970  		stat.merges[STAT_READ],
971  		(unsigned long long)stat.sectors[STAT_READ],
972  		(unsigned int)div_u64(stat.nsecs[STAT_READ], NSEC_PER_MSEC),
973  		stat.ios[STAT_WRITE],
974  		stat.merges[STAT_WRITE],
975  		(unsigned long long)stat.sectors[STAT_WRITE],
976  		(unsigned int)div_u64(stat.nsecs[STAT_WRITE], NSEC_PER_MSEC),
977  		inflight,
978  		jiffies_to_msecs(stat.io_ticks),
979  		(unsigned int)div_u64(stat.nsecs[STAT_READ] +
980  				      stat.nsecs[STAT_WRITE] +
981  				      stat.nsecs[STAT_DISCARD] +
982  				      stat.nsecs[STAT_FLUSH],
983  						NSEC_PER_MSEC),
984  		stat.ios[STAT_DISCARD],
985  		stat.merges[STAT_DISCARD],
986  		(unsigned long long)stat.sectors[STAT_DISCARD],
987  		(unsigned int)div_u64(stat.nsecs[STAT_DISCARD], NSEC_PER_MSEC),
988  		stat.ios[STAT_FLUSH],
989  		(unsigned int)div_u64(stat.nsecs[STAT_FLUSH], NSEC_PER_MSEC));
990  }
991  
992  ssize_t part_inflight_show(struct device *dev, struct device_attribute *attr,
993  			   char *buf)
994  {
995  	struct block_device *bdev = dev_to_bdev(dev);
996  	struct request_queue *q = bdev_get_queue(bdev);
997  	unsigned int inflight[2];
998  
999  	if (queue_is_mq(q))
1000  		blk_mq_in_flight_rw(q, bdev, inflight);
1001  	else
1002  		part_in_flight_rw(bdev, inflight);
1003  
1004  	return sprintf(buf, "%8u %8u\n", inflight[0], inflight[1]);
1005  }
1006  
1007  static ssize_t disk_capability_show(struct device *dev,
1008  				    struct device_attribute *attr, char *buf)
1009  {
1010  	dev_warn_once(dev, "the capability attribute has been deprecated.\n");
1011  	return sprintf(buf, "0\n");
1012  }
1013  
1014  static ssize_t disk_alignment_offset_show(struct device *dev,
1015  					  struct device_attribute *attr,
1016  					  char *buf)
1017  {
1018  	struct gendisk *disk = dev_to_disk(dev);
1019  
1020  	return sprintf(buf, "%d\n", bdev_alignment_offset(disk->part0));
1021  }
1022  
1023  static ssize_t disk_discard_alignment_show(struct device *dev,
1024  					   struct device_attribute *attr,
1025  					   char *buf)
1026  {
1027  	struct gendisk *disk = dev_to_disk(dev);
1028  
1029  	return sprintf(buf, "%d\n", bdev_alignment_offset(disk->part0));
1030  }
1031  
1032  static ssize_t diskseq_show(struct device *dev,
1033  			    struct device_attribute *attr, char *buf)
1034  {
1035  	struct gendisk *disk = dev_to_disk(dev);
1036  
1037  	return sprintf(buf, "%llu\n", disk->diskseq);
1038  }
1039  
1040  static DEVICE_ATTR(range, 0444, disk_range_show, NULL);
1041  static DEVICE_ATTR(ext_range, 0444, disk_ext_range_show, NULL);
1042  static DEVICE_ATTR(removable, 0444, disk_removable_show, NULL);
1043  static DEVICE_ATTR(hidden, 0444, disk_hidden_show, NULL);
1044  static DEVICE_ATTR(ro, 0444, disk_ro_show, NULL);
1045  static DEVICE_ATTR(size, 0444, part_size_show, NULL);
1046  static DEVICE_ATTR(alignment_offset, 0444, disk_alignment_offset_show, NULL);
1047  static DEVICE_ATTR(discard_alignment, 0444, disk_discard_alignment_show, NULL);
1048  static DEVICE_ATTR(capability, 0444, disk_capability_show, NULL);
1049  static DEVICE_ATTR(stat, 0444, part_stat_show, NULL);
1050  static DEVICE_ATTR(inflight, 0444, part_inflight_show, NULL);
1051  static DEVICE_ATTR(badblocks, 0644, disk_badblocks_show, disk_badblocks_store);
1052  static DEVICE_ATTR(diskseq, 0444, diskseq_show, NULL);
1053  
1054  #ifdef CONFIG_FAIL_MAKE_REQUEST
1055  ssize_t part_fail_show(struct device *dev,
1056  		       struct device_attribute *attr, char *buf)
1057  {
1058  	return sprintf(buf, "%d\n", dev_to_bdev(dev)->bd_make_it_fail);
1059  }
1060  
1061  ssize_t part_fail_store(struct device *dev,
1062  			struct device_attribute *attr,
1063  			const char *buf, size_t count)
1064  {
1065  	int i;
1066  
1067  	if (count > 0 && sscanf(buf, "%d", &i) > 0)
1068  		dev_to_bdev(dev)->bd_make_it_fail = i;
1069  
1070  	return count;
1071  }
1072  
1073  static struct device_attribute dev_attr_fail =
1074  	__ATTR(make-it-fail, 0644, part_fail_show, part_fail_store);
1075  #endif /* CONFIG_FAIL_MAKE_REQUEST */
1076  
1077  #ifdef CONFIG_FAIL_IO_TIMEOUT
1078  static struct device_attribute dev_attr_fail_timeout =
1079  	__ATTR(io-timeout-fail, 0644, part_timeout_show, part_timeout_store);
1080  #endif
1081  
1082  static struct attribute *disk_attrs[] = {
1083  	&dev_attr_range.attr,
1084  	&dev_attr_ext_range.attr,
1085  	&dev_attr_removable.attr,
1086  	&dev_attr_hidden.attr,
1087  	&dev_attr_ro.attr,
1088  	&dev_attr_size.attr,
1089  	&dev_attr_alignment_offset.attr,
1090  	&dev_attr_discard_alignment.attr,
1091  	&dev_attr_capability.attr,
1092  	&dev_attr_stat.attr,
1093  	&dev_attr_inflight.attr,
1094  	&dev_attr_badblocks.attr,
1095  	&dev_attr_events.attr,
1096  	&dev_attr_events_async.attr,
1097  	&dev_attr_events_poll_msecs.attr,
1098  	&dev_attr_diskseq.attr,
1099  #ifdef CONFIG_FAIL_MAKE_REQUEST
1100  	&dev_attr_fail.attr,
1101  #endif
1102  #ifdef CONFIG_FAIL_IO_TIMEOUT
1103  	&dev_attr_fail_timeout.attr,
1104  #endif
1105  	NULL
1106  };
1107  
1108  static umode_t disk_visible(struct kobject *kobj, struct attribute *a, int n)
1109  {
1110  	struct device *dev = container_of(kobj, typeof(*dev), kobj);
1111  	struct gendisk *disk = dev_to_disk(dev);
1112  
1113  	if (a == &dev_attr_badblocks.attr && !disk->bb)
1114  		return 0;
1115  	return a->mode;
1116  }
1117  
1118  static struct attribute_group disk_attr_group = {
1119  	.attrs = disk_attrs,
1120  	.is_visible = disk_visible,
1121  };
1122  
1123  static const struct attribute_group *disk_attr_groups[] = {
1124  	&disk_attr_group,
1125  #ifdef CONFIG_BLK_DEV_IO_TRACE
1126  	&blk_trace_attr_group,
1127  #endif
1128  #ifdef CONFIG_BLK_DEV_INTEGRITY
1129  	&blk_integrity_attr_group,
1130  #endif
1131  	NULL
1132  };
1133  
1134  /**
1135   * disk_release - releases all allocated resources of the gendisk
1136   * @dev: the device representing this disk
1137   *
1138   * This function releases all allocated resources of the gendisk.
1139   *
1140   * Drivers which used __device_add_disk() have a gendisk with a request_queue
1141   * assigned. Since the request_queue sits on top of the gendisk for these
1142   * drivers we also call blk_put_queue() for them, and we expect the
1143   * request_queue refcount to reach 0 at this point, and so the request_queue
1144   * will also be freed prior to the disk.
1145   *
1146   * Context: can sleep
1147   */
1148  static void disk_release(struct device *dev)
1149  {
1150  	struct gendisk *disk = dev_to_disk(dev);
1151  
1152  	might_sleep();
1153  	WARN_ON_ONCE(disk_live(disk));
1154  
1155  	blk_trace_remove(disk->queue);
1156  
1157  	/*
1158  	 * To undo the all initialization from blk_mq_init_allocated_queue in
1159  	 * case of a probe failure where add_disk is never called we have to
1160  	 * call blk_mq_exit_queue here. We can't do this for the more common
1161  	 * teardown case (yet) as the tagset can be gone by the time the disk
1162  	 * is released once it was added.
1163  	 */
1164  	if (queue_is_mq(disk->queue) &&
1165  	    test_bit(GD_OWNS_QUEUE, &disk->state) &&
1166  	    !test_bit(GD_ADDED, &disk->state))
1167  		blk_mq_exit_queue(disk->queue);
1168  
1169  	blkcg_exit_disk(disk);
1170  
1171  	bioset_exit(&disk->bio_split);
1172  
1173  	disk_release_events(disk);
1174  	kfree(disk->random);
1175  	disk_free_zone_bitmaps(disk);
1176  	xa_destroy(&disk->part_tbl);
1177  
1178  	disk->queue->disk = NULL;
1179  	blk_put_queue(disk->queue);
1180  
1181  	if (test_bit(GD_ADDED, &disk->state) && disk->fops->free_disk)
1182  		disk->fops->free_disk(disk);
1183  
1184  	iput(disk->part0->bd_inode);	/* frees the disk */
1185  }
1186  
1187  static int block_uevent(const struct device *dev, struct kobj_uevent_env *env)
1188  {
1189  	const struct gendisk *disk = dev_to_disk(dev);
1190  
1191  	return add_uevent_var(env, "DISKSEQ=%llu", disk->diskseq);
1192  }
1193  
1194  struct class block_class = {
1195  	.name		= "block",
1196  	.dev_uevent	= block_uevent,
1197  };
1198  
1199  static char *block_devnode(const struct device *dev, umode_t *mode,
1200  			   kuid_t *uid, kgid_t *gid)
1201  {
1202  	struct gendisk *disk = dev_to_disk(dev);
1203  
1204  	if (disk->fops->devnode)
1205  		return disk->fops->devnode(disk, mode);
1206  	return NULL;
1207  }
1208  
1209  const struct device_type disk_type = {
1210  	.name		= "disk",
1211  	.groups		= disk_attr_groups,
1212  	.release	= disk_release,
1213  	.devnode	= block_devnode,
1214  };
1215  
1216  #ifdef CONFIG_PROC_FS
1217  /*
1218   * aggregate disk stat collector.  Uses the same stats that the sysfs
1219   * entries do, above, but makes them available through one seq_file.
1220   *
1221   * The output looks suspiciously like /proc/partitions with a bunch of
1222   * extra fields.
1223   */
1224  static int diskstats_show(struct seq_file *seqf, void *v)
1225  {
1226  	struct gendisk *gp = v;
1227  	struct block_device *hd;
1228  	unsigned int inflight;
1229  	struct disk_stats stat;
1230  	unsigned long idx;
1231  
1232  	/*
1233  	if (&disk_to_dev(gp)->kobj.entry == block_class.devices.next)
1234  		seq_puts(seqf,	"major minor name"
1235  				"     rio rmerge rsect ruse wio wmerge "
1236  				"wsect wuse running use aveq"
1237  				"\n\n");
1238  	*/
1239  
1240  	rcu_read_lock();
1241  	xa_for_each(&gp->part_tbl, idx, hd) {
1242  		if (bdev_is_partition(hd) && !bdev_nr_sectors(hd))
1243  			continue;
1244  		if (queue_is_mq(gp->queue))
1245  			inflight = blk_mq_in_flight(gp->queue, hd);
1246  		else
1247  			inflight = part_in_flight(hd);
1248  
1249  		if (inflight) {
1250  			part_stat_lock();
1251  			update_io_ticks(hd, jiffies, true);
1252  			part_stat_unlock();
1253  		}
1254  		part_stat_read_all(hd, &stat);
1255  		seq_printf(seqf, "%4d %7d %pg "
1256  			   "%lu %lu %lu %u "
1257  			   "%lu %lu %lu %u "
1258  			   "%u %u %u "
1259  			   "%lu %lu %lu %u "
1260  			   "%lu %u"
1261  			   "\n",
1262  			   MAJOR(hd->bd_dev), MINOR(hd->bd_dev), hd,
1263  			   stat.ios[STAT_READ],
1264  			   stat.merges[STAT_READ],
1265  			   stat.sectors[STAT_READ],
1266  			   (unsigned int)div_u64(stat.nsecs[STAT_READ],
1267  							NSEC_PER_MSEC),
1268  			   stat.ios[STAT_WRITE],
1269  			   stat.merges[STAT_WRITE],
1270  			   stat.sectors[STAT_WRITE],
1271  			   (unsigned int)div_u64(stat.nsecs[STAT_WRITE],
1272  							NSEC_PER_MSEC),
1273  			   inflight,
1274  			   jiffies_to_msecs(stat.io_ticks),
1275  			   (unsigned int)div_u64(stat.nsecs[STAT_READ] +
1276  						 stat.nsecs[STAT_WRITE] +
1277  						 stat.nsecs[STAT_DISCARD] +
1278  						 stat.nsecs[STAT_FLUSH],
1279  							NSEC_PER_MSEC),
1280  			   stat.ios[STAT_DISCARD],
1281  			   stat.merges[STAT_DISCARD],
1282  			   stat.sectors[STAT_DISCARD],
1283  			   (unsigned int)div_u64(stat.nsecs[STAT_DISCARD],
1284  						 NSEC_PER_MSEC),
1285  			   stat.ios[STAT_FLUSH],
1286  			   (unsigned int)div_u64(stat.nsecs[STAT_FLUSH],
1287  						 NSEC_PER_MSEC)
1288  			);
1289  	}
1290  	rcu_read_unlock();
1291  
1292  	return 0;
1293  }
1294  
1295  static const struct seq_operations diskstats_op = {
1296  	.start	= disk_seqf_start,
1297  	.next	= disk_seqf_next,
1298  	.stop	= disk_seqf_stop,
1299  	.show	= diskstats_show
1300  };
1301  
1302  static int __init proc_genhd_init(void)
1303  {
1304  	proc_create_seq("diskstats", 0, NULL, &diskstats_op);
1305  	proc_create_seq("partitions", 0, NULL, &partitions_op);
1306  	return 0;
1307  }
1308  module_init(proc_genhd_init);
1309  #endif /* CONFIG_PROC_FS */
1310  
1311  dev_t part_devt(struct gendisk *disk, u8 partno)
1312  {
1313  	struct block_device *part;
1314  	dev_t devt = 0;
1315  
1316  	rcu_read_lock();
1317  	part = xa_load(&disk->part_tbl, partno);
1318  	if (part)
1319  		devt = part->bd_dev;
1320  	rcu_read_unlock();
1321  
1322  	return devt;
1323  }
1324  
1325  struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id,
1326  		struct lock_class_key *lkclass)
1327  {
1328  	struct gendisk *disk;
1329  
1330  	disk = kzalloc_node(sizeof(struct gendisk), GFP_KERNEL, node_id);
1331  	if (!disk)
1332  		return NULL;
1333  
1334  	if (bioset_init(&disk->bio_split, BIO_POOL_SIZE, 0, 0))
1335  		goto out_free_disk;
1336  
1337  	disk->bdi = bdi_alloc(node_id);
1338  	if (!disk->bdi)
1339  		goto out_free_bioset;
1340  
1341  	/* bdev_alloc() might need the queue, set before the first call */
1342  	disk->queue = q;
1343  
1344  	disk->part0 = bdev_alloc(disk, 0);
1345  	if (!disk->part0)
1346  		goto out_free_bdi;
1347  
1348  	disk->node_id = node_id;
1349  	mutex_init(&disk->open_mutex);
1350  	xa_init(&disk->part_tbl);
1351  	if (xa_insert(&disk->part_tbl, 0, disk->part0, GFP_KERNEL))
1352  		goto out_destroy_part_tbl;
1353  
1354  	if (blkcg_init_disk(disk))
1355  		goto out_erase_part0;
1356  
1357  	rand_initialize_disk(disk);
1358  	disk_to_dev(disk)->class = &block_class;
1359  	disk_to_dev(disk)->type = &disk_type;
1360  	device_initialize(disk_to_dev(disk));
1361  	inc_diskseq(disk);
1362  	q->disk = disk;
1363  	lockdep_init_map(&disk->lockdep_map, "(bio completion)", lkclass, 0);
1364  #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED
1365  	INIT_LIST_HEAD(&disk->slave_bdevs);
1366  #endif
1367  	return disk;
1368  
1369  out_erase_part0:
1370  	xa_erase(&disk->part_tbl, 0);
1371  out_destroy_part_tbl:
1372  	xa_destroy(&disk->part_tbl);
1373  	disk->part0->bd_disk = NULL;
1374  	iput(disk->part0->bd_inode);
1375  out_free_bdi:
1376  	bdi_put(disk->bdi);
1377  out_free_bioset:
1378  	bioset_exit(&disk->bio_split);
1379  out_free_disk:
1380  	kfree(disk);
1381  	return NULL;
1382  }
1383  
1384  struct gendisk *__blk_alloc_disk(int node, struct lock_class_key *lkclass)
1385  {
1386  	struct request_queue *q;
1387  	struct gendisk *disk;
1388  
1389  	q = blk_alloc_queue(node);
1390  	if (!q)
1391  		return NULL;
1392  
1393  	disk = __alloc_disk_node(q, node, lkclass);
1394  	if (!disk) {
1395  		blk_put_queue(q);
1396  		return NULL;
1397  	}
1398  	set_bit(GD_OWNS_QUEUE, &disk->state);
1399  	return disk;
1400  }
1401  EXPORT_SYMBOL(__blk_alloc_disk);
1402  
1403  /**
1404   * put_disk - decrements the gendisk refcount
1405   * @disk: the struct gendisk to decrement the refcount for
1406   *
1407   * This decrements the refcount for the struct gendisk. When this reaches 0
1408   * we'll have disk_release() called.
1409   *
1410   * Note: for blk-mq disk put_disk must be called before freeing the tag_set
1411   * when handling probe errors (that is before add_disk() is called).
1412   *
1413   * Context: Any context, but the last reference must not be dropped from
1414   *          atomic context.
1415   */
1416  void put_disk(struct gendisk *disk)
1417  {
1418  	if (disk)
1419  		put_device(disk_to_dev(disk));
1420  }
1421  EXPORT_SYMBOL(put_disk);
1422  
1423  static void set_disk_ro_uevent(struct gendisk *gd, int ro)
1424  {
1425  	char event[] = "DISK_RO=1";
1426  	char *envp[] = { event, NULL };
1427  
1428  	if (!ro)
1429  		event[8] = '0';
1430  	kobject_uevent_env(&disk_to_dev(gd)->kobj, KOBJ_CHANGE, envp);
1431  }
1432  
1433  /**
1434   * set_disk_ro - set a gendisk read-only
1435   * @disk:	gendisk to operate on
1436   * @read_only:	%true to set the disk read-only, %false set the disk read/write
1437   *
1438   * This function is used to indicate whether a given disk device should have its
1439   * read-only flag set. set_disk_ro() is typically used by device drivers to
1440   * indicate whether the underlying physical device is write-protected.
1441   */
1442  void set_disk_ro(struct gendisk *disk, bool read_only)
1443  {
1444  	if (read_only) {
1445  		if (test_and_set_bit(GD_READ_ONLY, &disk->state))
1446  			return;
1447  	} else {
1448  		if (!test_and_clear_bit(GD_READ_ONLY, &disk->state))
1449  			return;
1450  	}
1451  	set_disk_ro_uevent(disk, read_only);
1452  }
1453  EXPORT_SYMBOL(set_disk_ro);
1454  
1455  void inc_diskseq(struct gendisk *disk)
1456  {
1457  	disk->diskseq = atomic64_inc_return(&diskseq);
1458  }
1459