xref: /openbmc/linux/block/genhd.c (revision e2f1cf25)
1 /*
2  *  gendisk handling
3  */
4 
5 #include <linux/module.h>
6 #include <linux/fs.h>
7 #include <linux/genhd.h>
8 #include <linux/kdev_t.h>
9 #include <linux/kernel.h>
10 #include <linux/blkdev.h>
11 #include <linux/backing-dev.h>
12 #include <linux/init.h>
13 #include <linux/spinlock.h>
14 #include <linux/proc_fs.h>
15 #include <linux/seq_file.h>
16 #include <linux/slab.h>
17 #include <linux/kmod.h>
18 #include <linux/kobj_map.h>
19 #include <linux/mutex.h>
20 #include <linux/idr.h>
21 #include <linux/log2.h>
22 #include <linux/pm_runtime.h>
23 
24 #include "blk.h"
25 
26 static DEFINE_MUTEX(block_class_lock);
27 struct kobject *block_depr;
28 
29 /* for extended dynamic devt allocation, currently only one major is used */
30 #define NR_EXT_DEVT		(1 << MINORBITS)
31 
32 /* For extended devt allocation.  ext_devt_lock prevents look up
33  * results from going away underneath its user.
34  */
35 static DEFINE_SPINLOCK(ext_devt_lock);
36 static DEFINE_IDR(ext_devt_idr);
37 
38 static struct device_type disk_type;
39 
40 static void disk_check_events(struct disk_events *ev,
41 			      unsigned int *clearing_ptr);
42 static void disk_alloc_events(struct gendisk *disk);
43 static void disk_add_events(struct gendisk *disk);
44 static void disk_del_events(struct gendisk *disk);
45 static void disk_release_events(struct gendisk *disk);
46 
47 /**
48  * disk_get_part - get partition
49  * @disk: disk to look partition from
50  * @partno: partition number
51  *
52  * Look for partition @partno from @disk.  If found, increment
53  * reference count and return it.
54  *
55  * CONTEXT:
56  * Don't care.
57  *
58  * RETURNS:
59  * Pointer to the found partition on success, NULL if not found.
60  */
61 struct hd_struct *disk_get_part(struct gendisk *disk, int partno)
62 {
63 	struct hd_struct *part = NULL;
64 	struct disk_part_tbl *ptbl;
65 
66 	if (unlikely(partno < 0))
67 		return NULL;
68 
69 	rcu_read_lock();
70 
71 	ptbl = rcu_dereference(disk->part_tbl);
72 	if (likely(partno < ptbl->len)) {
73 		part = rcu_dereference(ptbl->part[partno]);
74 		if (part)
75 			get_device(part_to_dev(part));
76 	}
77 
78 	rcu_read_unlock();
79 
80 	return part;
81 }
82 EXPORT_SYMBOL_GPL(disk_get_part);
83 
84 /**
85  * disk_part_iter_init - initialize partition iterator
86  * @piter: iterator to initialize
87  * @disk: disk to iterate over
88  * @flags: DISK_PITER_* flags
89  *
90  * Initialize @piter so that it iterates over partitions of @disk.
91  *
92  * CONTEXT:
93  * Don't care.
94  */
95 void disk_part_iter_init(struct disk_part_iter *piter, struct gendisk *disk,
96 			  unsigned int flags)
97 {
98 	struct disk_part_tbl *ptbl;
99 
100 	rcu_read_lock();
101 	ptbl = rcu_dereference(disk->part_tbl);
102 
103 	piter->disk = disk;
104 	piter->part = NULL;
105 
106 	if (flags & DISK_PITER_REVERSE)
107 		piter->idx = ptbl->len - 1;
108 	else if (flags & (DISK_PITER_INCL_PART0 | DISK_PITER_INCL_EMPTY_PART0))
109 		piter->idx = 0;
110 	else
111 		piter->idx = 1;
112 
113 	piter->flags = flags;
114 
115 	rcu_read_unlock();
116 }
117 EXPORT_SYMBOL_GPL(disk_part_iter_init);
118 
119 /**
120  * disk_part_iter_next - proceed iterator to the next partition and return it
121  * @piter: iterator of interest
122  *
123  * Proceed @piter to the next partition and return it.
124  *
125  * CONTEXT:
126  * Don't care.
127  */
128 struct hd_struct *disk_part_iter_next(struct disk_part_iter *piter)
129 {
130 	struct disk_part_tbl *ptbl;
131 	int inc, end;
132 
133 	/* put the last partition */
134 	disk_put_part(piter->part);
135 	piter->part = NULL;
136 
137 	/* get part_tbl */
138 	rcu_read_lock();
139 	ptbl = rcu_dereference(piter->disk->part_tbl);
140 
141 	/* determine iteration parameters */
142 	if (piter->flags & DISK_PITER_REVERSE) {
143 		inc = -1;
144 		if (piter->flags & (DISK_PITER_INCL_PART0 |
145 				    DISK_PITER_INCL_EMPTY_PART0))
146 			end = -1;
147 		else
148 			end = 0;
149 	} else {
150 		inc = 1;
151 		end = ptbl->len;
152 	}
153 
154 	/* iterate to the next partition */
155 	for (; piter->idx != end; piter->idx += inc) {
156 		struct hd_struct *part;
157 
158 		part = rcu_dereference(ptbl->part[piter->idx]);
159 		if (!part)
160 			continue;
161 		if (!part_nr_sects_read(part) &&
162 		    !(piter->flags & DISK_PITER_INCL_EMPTY) &&
163 		    !(piter->flags & DISK_PITER_INCL_EMPTY_PART0 &&
164 		      piter->idx == 0))
165 			continue;
166 
167 		get_device(part_to_dev(part));
168 		piter->part = part;
169 		piter->idx += inc;
170 		break;
171 	}
172 
173 	rcu_read_unlock();
174 
175 	return piter->part;
176 }
177 EXPORT_SYMBOL_GPL(disk_part_iter_next);
178 
179 /**
180  * disk_part_iter_exit - finish up partition iteration
181  * @piter: iter of interest
182  *
183  * Called when iteration is over.  Cleans up @piter.
184  *
185  * CONTEXT:
186  * Don't care.
187  */
188 void disk_part_iter_exit(struct disk_part_iter *piter)
189 {
190 	disk_put_part(piter->part);
191 	piter->part = NULL;
192 }
193 EXPORT_SYMBOL_GPL(disk_part_iter_exit);
194 
195 static inline int sector_in_part(struct hd_struct *part, sector_t sector)
196 {
197 	return part->start_sect <= sector &&
198 		sector < part->start_sect + part_nr_sects_read(part);
199 }
200 
201 /**
202  * disk_map_sector_rcu - map sector to partition
203  * @disk: gendisk of interest
204  * @sector: sector to map
205  *
206  * Find out which partition @sector maps to on @disk.  This is
207  * primarily used for stats accounting.
208  *
209  * CONTEXT:
210  * RCU read locked.  The returned partition pointer is valid only
211  * while preemption is disabled.
212  *
213  * RETURNS:
214  * Found partition on success, part0 is returned if no partition matches
215  */
216 struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector)
217 {
218 	struct disk_part_tbl *ptbl;
219 	struct hd_struct *part;
220 	int i;
221 
222 	ptbl = rcu_dereference(disk->part_tbl);
223 
224 	part = rcu_dereference(ptbl->last_lookup);
225 	if (part && sector_in_part(part, sector))
226 		return part;
227 
228 	for (i = 1; i < ptbl->len; i++) {
229 		part = rcu_dereference(ptbl->part[i]);
230 
231 		if (part && sector_in_part(part, sector)) {
232 			rcu_assign_pointer(ptbl->last_lookup, part);
233 			return part;
234 		}
235 	}
236 	return &disk->part0;
237 }
238 EXPORT_SYMBOL_GPL(disk_map_sector_rcu);
239 
240 /*
241  * Can be deleted altogether. Later.
242  *
243  */
244 static struct blk_major_name {
245 	struct blk_major_name *next;
246 	int major;
247 	char name[16];
248 } *major_names[BLKDEV_MAJOR_HASH_SIZE];
249 
250 /* index in the above - for now: assume no multimajor ranges */
251 static inline int major_to_index(unsigned major)
252 {
253 	return major % BLKDEV_MAJOR_HASH_SIZE;
254 }
255 
256 #ifdef CONFIG_PROC_FS
257 void blkdev_show(struct seq_file *seqf, off_t offset)
258 {
259 	struct blk_major_name *dp;
260 
261 	if (offset < BLKDEV_MAJOR_HASH_SIZE) {
262 		mutex_lock(&block_class_lock);
263 		for (dp = major_names[offset]; dp; dp = dp->next)
264 			seq_printf(seqf, "%3d %s\n", dp->major, dp->name);
265 		mutex_unlock(&block_class_lock);
266 	}
267 }
268 #endif /* CONFIG_PROC_FS */
269 
270 /**
271  * register_blkdev - register a new block device
272  *
273  * @major: the requested major device number [1..255]. If @major=0, try to
274  *         allocate any unused major number.
275  * @name: the name of the new block device as a zero terminated string
276  *
277  * The @name must be unique within the system.
278  *
279  * The return value depends on the @major input parameter.
280  *  - if a major device number was requested in range [1..255] then the
281  *    function returns zero on success, or a negative error code
282  *  - if any unused major number was requested with @major=0 parameter
283  *    then the return value is the allocated major number in range
284  *    [1..255] or a negative error code otherwise
285  */
286 int register_blkdev(unsigned int major, const char *name)
287 {
288 	struct blk_major_name **n, *p;
289 	int index, ret = 0;
290 
291 	mutex_lock(&block_class_lock);
292 
293 	/* temporary */
294 	if (major == 0) {
295 		for (index = ARRAY_SIZE(major_names)-1; index > 0; index--) {
296 			if (major_names[index] == NULL)
297 				break;
298 		}
299 
300 		if (index == 0) {
301 			printk("register_blkdev: failed to get major for %s\n",
302 			       name);
303 			ret = -EBUSY;
304 			goto out;
305 		}
306 		major = index;
307 		ret = major;
308 	}
309 
310 	p = kmalloc(sizeof(struct blk_major_name), GFP_KERNEL);
311 	if (p == NULL) {
312 		ret = -ENOMEM;
313 		goto out;
314 	}
315 
316 	p->major = major;
317 	strlcpy(p->name, name, sizeof(p->name));
318 	p->next = NULL;
319 	index = major_to_index(major);
320 
321 	for (n = &major_names[index]; *n; n = &(*n)->next) {
322 		if ((*n)->major == major)
323 			break;
324 	}
325 	if (!*n)
326 		*n = p;
327 	else
328 		ret = -EBUSY;
329 
330 	if (ret < 0) {
331 		printk("register_blkdev: cannot get major %d for %s\n",
332 		       major, name);
333 		kfree(p);
334 	}
335 out:
336 	mutex_unlock(&block_class_lock);
337 	return ret;
338 }
339 
340 EXPORT_SYMBOL(register_blkdev);
341 
342 void unregister_blkdev(unsigned int major, const char *name)
343 {
344 	struct blk_major_name **n;
345 	struct blk_major_name *p = NULL;
346 	int index = major_to_index(major);
347 
348 	mutex_lock(&block_class_lock);
349 	for (n = &major_names[index]; *n; n = &(*n)->next)
350 		if ((*n)->major == major)
351 			break;
352 	if (!*n || strcmp((*n)->name, name)) {
353 		WARN_ON(1);
354 	} else {
355 		p = *n;
356 		*n = p->next;
357 	}
358 	mutex_unlock(&block_class_lock);
359 	kfree(p);
360 }
361 
362 EXPORT_SYMBOL(unregister_blkdev);
363 
364 static struct kobj_map *bdev_map;
365 
366 /**
367  * blk_mangle_minor - scatter minor numbers apart
368  * @minor: minor number to mangle
369  *
370  * Scatter consecutively allocated @minor number apart if MANGLE_DEVT
371  * is enabled.  Mangling twice gives the original value.
372  *
373  * RETURNS:
374  * Mangled value.
375  *
376  * CONTEXT:
377  * Don't care.
378  */
379 static int blk_mangle_minor(int minor)
380 {
381 #ifdef CONFIG_DEBUG_BLOCK_EXT_DEVT
382 	int i;
383 
384 	for (i = 0; i < MINORBITS / 2; i++) {
385 		int low = minor & (1 << i);
386 		int high = minor & (1 << (MINORBITS - 1 - i));
387 		int distance = MINORBITS - 1 - 2 * i;
388 
389 		minor ^= low | high;	/* clear both bits */
390 		low <<= distance;	/* swap the positions */
391 		high >>= distance;
392 		minor |= low | high;	/* and set */
393 	}
394 #endif
395 	return minor;
396 }
397 
398 /**
399  * blk_alloc_devt - allocate a dev_t for a partition
400  * @part: partition to allocate dev_t for
401  * @devt: out parameter for resulting dev_t
402  *
403  * Allocate a dev_t for block device.
404  *
405  * RETURNS:
406  * 0 on success, allocated dev_t is returned in *@devt.  -errno on
407  * failure.
408  *
409  * CONTEXT:
410  * Might sleep.
411  */
412 int blk_alloc_devt(struct hd_struct *part, dev_t *devt)
413 {
414 	struct gendisk *disk = part_to_disk(part);
415 	int idx;
416 
417 	/* in consecutive minor range? */
418 	if (part->partno < disk->minors) {
419 		*devt = MKDEV(disk->major, disk->first_minor + part->partno);
420 		return 0;
421 	}
422 
423 	/* allocate ext devt */
424 	idr_preload(GFP_KERNEL);
425 
426 	spin_lock_bh(&ext_devt_lock);
427 	idx = idr_alloc(&ext_devt_idr, part, 0, NR_EXT_DEVT, GFP_NOWAIT);
428 	spin_unlock_bh(&ext_devt_lock);
429 
430 	idr_preload_end();
431 	if (idx < 0)
432 		return idx == -ENOSPC ? -EBUSY : idx;
433 
434 	*devt = MKDEV(BLOCK_EXT_MAJOR, blk_mangle_minor(idx));
435 	return 0;
436 }
437 
438 /**
439  * blk_free_devt - free a dev_t
440  * @devt: dev_t to free
441  *
442  * Free @devt which was allocated using blk_alloc_devt().
443  *
444  * CONTEXT:
445  * Might sleep.
446  */
447 void blk_free_devt(dev_t devt)
448 {
449 	if (devt == MKDEV(0, 0))
450 		return;
451 
452 	if (MAJOR(devt) == BLOCK_EXT_MAJOR) {
453 		spin_lock_bh(&ext_devt_lock);
454 		idr_remove(&ext_devt_idr, blk_mangle_minor(MINOR(devt)));
455 		spin_unlock_bh(&ext_devt_lock);
456 	}
457 }
458 
459 static char *bdevt_str(dev_t devt, char *buf)
460 {
461 	if (MAJOR(devt) <= 0xff && MINOR(devt) <= 0xff) {
462 		char tbuf[BDEVT_SIZE];
463 		snprintf(tbuf, BDEVT_SIZE, "%02x%02x", MAJOR(devt), MINOR(devt));
464 		snprintf(buf, BDEVT_SIZE, "%-9s", tbuf);
465 	} else
466 		snprintf(buf, BDEVT_SIZE, "%03x:%05x", MAJOR(devt), MINOR(devt));
467 
468 	return buf;
469 }
470 
471 /*
472  * Register device numbers dev..(dev+range-1)
473  * range must be nonzero
474  * The hash chain is sorted on range, so that subranges can override.
475  */
476 void blk_register_region(dev_t devt, unsigned long range, struct module *module,
477 			 struct kobject *(*probe)(dev_t, int *, void *),
478 			 int (*lock)(dev_t, void *), void *data)
479 {
480 	kobj_map(bdev_map, devt, range, module, probe, lock, data);
481 }
482 
483 EXPORT_SYMBOL(blk_register_region);
484 
485 void blk_unregister_region(dev_t devt, unsigned long range)
486 {
487 	kobj_unmap(bdev_map, devt, range);
488 }
489 
490 EXPORT_SYMBOL(blk_unregister_region);
491 
492 static struct kobject *exact_match(dev_t devt, int *partno, void *data)
493 {
494 	struct gendisk *p = data;
495 
496 	return &disk_to_dev(p)->kobj;
497 }
498 
499 static int exact_lock(dev_t devt, void *data)
500 {
501 	struct gendisk *p = data;
502 
503 	if (!get_disk(p))
504 		return -1;
505 	return 0;
506 }
507 
508 static void register_disk(struct gendisk *disk)
509 {
510 	struct device *ddev = disk_to_dev(disk);
511 	struct block_device *bdev;
512 	struct disk_part_iter piter;
513 	struct hd_struct *part;
514 	int err;
515 
516 	ddev->parent = disk->driverfs_dev;
517 
518 	dev_set_name(ddev, "%s", disk->disk_name);
519 
520 	/* delay uevents, until we scanned partition table */
521 	dev_set_uevent_suppress(ddev, 1);
522 
523 	if (device_add(ddev))
524 		return;
525 	if (!sysfs_deprecated) {
526 		err = sysfs_create_link(block_depr, &ddev->kobj,
527 					kobject_name(&ddev->kobj));
528 		if (err) {
529 			device_del(ddev);
530 			return;
531 		}
532 	}
533 
534 	/*
535 	 * avoid probable deadlock caused by allocating memory with
536 	 * GFP_KERNEL in runtime_resume callback of its all ancestor
537 	 * devices
538 	 */
539 	pm_runtime_set_memalloc_noio(ddev, true);
540 
541 	disk->part0.holder_dir = kobject_create_and_add("holders", &ddev->kobj);
542 	disk->slave_dir = kobject_create_and_add("slaves", &ddev->kobj);
543 
544 	/* No minors to use for partitions */
545 	if (!disk_part_scan_enabled(disk))
546 		goto exit;
547 
548 	/* No such device (e.g., media were just removed) */
549 	if (!get_capacity(disk))
550 		goto exit;
551 
552 	bdev = bdget_disk(disk, 0);
553 	if (!bdev)
554 		goto exit;
555 
556 	bdev->bd_invalidated = 1;
557 	err = blkdev_get(bdev, FMODE_READ, NULL);
558 	if (err < 0)
559 		goto exit;
560 	blkdev_put(bdev, FMODE_READ);
561 
562 exit:
563 	/* announce disk after possible partitions are created */
564 	dev_set_uevent_suppress(ddev, 0);
565 	kobject_uevent(&ddev->kobj, KOBJ_ADD);
566 
567 	/* announce possible partitions */
568 	disk_part_iter_init(&piter, disk, 0);
569 	while ((part = disk_part_iter_next(&piter)))
570 		kobject_uevent(&part_to_dev(part)->kobj, KOBJ_ADD);
571 	disk_part_iter_exit(&piter);
572 }
573 
574 /**
575  * add_disk - add partitioning information to kernel list
576  * @disk: per-device partitioning information
577  *
578  * This function registers the partitioning information in @disk
579  * with the kernel.
580  *
581  * FIXME: error handling
582  */
583 void add_disk(struct gendisk *disk)
584 {
585 	struct backing_dev_info *bdi;
586 	dev_t devt;
587 	int retval;
588 
589 	/* minors == 0 indicates to use ext devt from part0 and should
590 	 * be accompanied with EXT_DEVT flag.  Make sure all
591 	 * parameters make sense.
592 	 */
593 	WARN_ON(disk->minors && !(disk->major || disk->first_minor));
594 	WARN_ON(!disk->minors && !(disk->flags & GENHD_FL_EXT_DEVT));
595 
596 	disk->flags |= GENHD_FL_UP;
597 
598 	retval = blk_alloc_devt(&disk->part0, &devt);
599 	if (retval) {
600 		WARN_ON(1);
601 		return;
602 	}
603 	disk_to_dev(disk)->devt = devt;
604 
605 	/* ->major and ->first_minor aren't supposed to be
606 	 * dereferenced from here on, but set them just in case.
607 	 */
608 	disk->major = MAJOR(devt);
609 	disk->first_minor = MINOR(devt);
610 
611 	disk_alloc_events(disk);
612 
613 	/* Register BDI before referencing it from bdev */
614 	bdi = &disk->queue->backing_dev_info;
615 	bdi_register_dev(bdi, disk_devt(disk));
616 
617 	blk_register_region(disk_devt(disk), disk->minors, NULL,
618 			    exact_match, exact_lock, disk);
619 	register_disk(disk);
620 	blk_register_queue(disk);
621 
622 	/*
623 	 * Take an extra ref on queue which will be put on disk_release()
624 	 * so that it sticks around as long as @disk is there.
625 	 */
626 	WARN_ON_ONCE(!blk_get_queue(disk->queue));
627 
628 	retval = sysfs_create_link(&disk_to_dev(disk)->kobj, &bdi->dev->kobj,
629 				   "bdi");
630 	WARN_ON(retval);
631 
632 	disk_add_events(disk);
633 }
634 EXPORT_SYMBOL(add_disk);
635 
636 void del_gendisk(struct gendisk *disk)
637 {
638 	struct disk_part_iter piter;
639 	struct hd_struct *part;
640 
641 	disk_del_events(disk);
642 
643 	/* invalidate stuff */
644 	disk_part_iter_init(&piter, disk,
645 			     DISK_PITER_INCL_EMPTY | DISK_PITER_REVERSE);
646 	while ((part = disk_part_iter_next(&piter))) {
647 		invalidate_partition(disk, part->partno);
648 		delete_partition(disk, part->partno);
649 	}
650 	disk_part_iter_exit(&piter);
651 
652 	invalidate_partition(disk, 0);
653 	set_capacity(disk, 0);
654 	disk->flags &= ~GENHD_FL_UP;
655 
656 	sysfs_remove_link(&disk_to_dev(disk)->kobj, "bdi");
657 	blk_unregister_queue(disk);
658 	blk_unregister_region(disk_devt(disk), disk->minors);
659 
660 	part_stat_set_all(&disk->part0, 0);
661 	disk->part0.stamp = 0;
662 
663 	kobject_put(disk->part0.holder_dir);
664 	kobject_put(disk->slave_dir);
665 	disk->driverfs_dev = NULL;
666 	if (!sysfs_deprecated)
667 		sysfs_remove_link(block_depr, dev_name(disk_to_dev(disk)));
668 	pm_runtime_set_memalloc_noio(disk_to_dev(disk), false);
669 	device_del(disk_to_dev(disk));
670 }
671 EXPORT_SYMBOL(del_gendisk);
672 
673 /**
674  * get_gendisk - get partitioning information for a given device
675  * @devt: device to get partitioning information for
676  * @partno: returned partition index
677  *
678  * This function gets the structure containing partitioning
679  * information for the given device @devt.
680  */
681 struct gendisk *get_gendisk(dev_t devt, int *partno)
682 {
683 	struct gendisk *disk = NULL;
684 
685 	if (MAJOR(devt) != BLOCK_EXT_MAJOR) {
686 		struct kobject *kobj;
687 
688 		kobj = kobj_lookup(bdev_map, devt, partno);
689 		if (kobj)
690 			disk = dev_to_disk(kobj_to_dev(kobj));
691 	} else {
692 		struct hd_struct *part;
693 
694 		spin_lock_bh(&ext_devt_lock);
695 		part = idr_find(&ext_devt_idr, blk_mangle_minor(MINOR(devt)));
696 		if (part && get_disk(part_to_disk(part))) {
697 			*partno = part->partno;
698 			disk = part_to_disk(part);
699 		}
700 		spin_unlock_bh(&ext_devt_lock);
701 	}
702 
703 	return disk;
704 }
705 EXPORT_SYMBOL(get_gendisk);
706 
707 /**
708  * bdget_disk - do bdget() by gendisk and partition number
709  * @disk: gendisk of interest
710  * @partno: partition number
711  *
712  * Find partition @partno from @disk, do bdget() on it.
713  *
714  * CONTEXT:
715  * Don't care.
716  *
717  * RETURNS:
718  * Resulting block_device on success, NULL on failure.
719  */
720 struct block_device *bdget_disk(struct gendisk *disk, int partno)
721 {
722 	struct hd_struct *part;
723 	struct block_device *bdev = NULL;
724 
725 	part = disk_get_part(disk, partno);
726 	if (part)
727 		bdev = bdget(part_devt(part));
728 	disk_put_part(part);
729 
730 	return bdev;
731 }
732 EXPORT_SYMBOL(bdget_disk);
733 
734 /*
735  * print a full list of all partitions - intended for places where the root
736  * filesystem can't be mounted and thus to give the victim some idea of what
737  * went wrong
738  */
739 void __init printk_all_partitions(void)
740 {
741 	struct class_dev_iter iter;
742 	struct device *dev;
743 
744 	class_dev_iter_init(&iter, &block_class, NULL, &disk_type);
745 	while ((dev = class_dev_iter_next(&iter))) {
746 		struct gendisk *disk = dev_to_disk(dev);
747 		struct disk_part_iter piter;
748 		struct hd_struct *part;
749 		char name_buf[BDEVNAME_SIZE];
750 		char devt_buf[BDEVT_SIZE];
751 
752 		/*
753 		 * Don't show empty devices or things that have been
754 		 * suppressed
755 		 */
756 		if (get_capacity(disk) == 0 ||
757 		    (disk->flags & GENHD_FL_SUPPRESS_PARTITION_INFO))
758 			continue;
759 
760 		/*
761 		 * Note, unlike /proc/partitions, I am showing the
762 		 * numbers in hex - the same format as the root=
763 		 * option takes.
764 		 */
765 		disk_part_iter_init(&piter, disk, DISK_PITER_INCL_PART0);
766 		while ((part = disk_part_iter_next(&piter))) {
767 			bool is_part0 = part == &disk->part0;
768 
769 			printk("%s%s %10llu %s %s", is_part0 ? "" : "  ",
770 			       bdevt_str(part_devt(part), devt_buf),
771 			       (unsigned long long)part_nr_sects_read(part) >> 1
772 			       , disk_name(disk, part->partno, name_buf),
773 			       part->info ? part->info->uuid : "");
774 			if (is_part0) {
775 				if (disk->driverfs_dev != NULL &&
776 				    disk->driverfs_dev->driver != NULL)
777 					printk(" driver: %s\n",
778 					      disk->driverfs_dev->driver->name);
779 				else
780 					printk(" (driver?)\n");
781 			} else
782 				printk("\n");
783 		}
784 		disk_part_iter_exit(&piter);
785 	}
786 	class_dev_iter_exit(&iter);
787 }
788 
789 #ifdef CONFIG_PROC_FS
790 /* iterator */
791 static void *disk_seqf_start(struct seq_file *seqf, loff_t *pos)
792 {
793 	loff_t skip = *pos;
794 	struct class_dev_iter *iter;
795 	struct device *dev;
796 
797 	iter = kmalloc(sizeof(*iter), GFP_KERNEL);
798 	if (!iter)
799 		return ERR_PTR(-ENOMEM);
800 
801 	seqf->private = iter;
802 	class_dev_iter_init(iter, &block_class, NULL, &disk_type);
803 	do {
804 		dev = class_dev_iter_next(iter);
805 		if (!dev)
806 			return NULL;
807 	} while (skip--);
808 
809 	return dev_to_disk(dev);
810 }
811 
812 static void *disk_seqf_next(struct seq_file *seqf, void *v, loff_t *pos)
813 {
814 	struct device *dev;
815 
816 	(*pos)++;
817 	dev = class_dev_iter_next(seqf->private);
818 	if (dev)
819 		return dev_to_disk(dev);
820 
821 	return NULL;
822 }
823 
824 static void disk_seqf_stop(struct seq_file *seqf, void *v)
825 {
826 	struct class_dev_iter *iter = seqf->private;
827 
828 	/* stop is called even after start failed :-( */
829 	if (iter) {
830 		class_dev_iter_exit(iter);
831 		kfree(iter);
832 	}
833 }
834 
835 static void *show_partition_start(struct seq_file *seqf, loff_t *pos)
836 {
837 	void *p;
838 
839 	p = disk_seqf_start(seqf, pos);
840 	if (!IS_ERR_OR_NULL(p) && !*pos)
841 		seq_puts(seqf, "major minor  #blocks  name\n\n");
842 	return p;
843 }
844 
845 static int show_partition(struct seq_file *seqf, void *v)
846 {
847 	struct gendisk *sgp = v;
848 	struct disk_part_iter piter;
849 	struct hd_struct *part;
850 	char buf[BDEVNAME_SIZE];
851 
852 	/* Don't show non-partitionable removeable devices or empty devices */
853 	if (!get_capacity(sgp) || (!disk_max_parts(sgp) &&
854 				   (sgp->flags & GENHD_FL_REMOVABLE)))
855 		return 0;
856 	if (sgp->flags & GENHD_FL_SUPPRESS_PARTITION_INFO)
857 		return 0;
858 
859 	/* show the full disk and all non-0 size partitions of it */
860 	disk_part_iter_init(&piter, sgp, DISK_PITER_INCL_PART0);
861 	while ((part = disk_part_iter_next(&piter)))
862 		seq_printf(seqf, "%4d  %7d %10llu %s\n",
863 			   MAJOR(part_devt(part)), MINOR(part_devt(part)),
864 			   (unsigned long long)part_nr_sects_read(part) >> 1,
865 			   disk_name(sgp, part->partno, buf));
866 	disk_part_iter_exit(&piter);
867 
868 	return 0;
869 }
870 
871 static const struct seq_operations partitions_op = {
872 	.start	= show_partition_start,
873 	.next	= disk_seqf_next,
874 	.stop	= disk_seqf_stop,
875 	.show	= show_partition
876 };
877 
878 static int partitions_open(struct inode *inode, struct file *file)
879 {
880 	return seq_open(file, &partitions_op);
881 }
882 
883 static const struct file_operations proc_partitions_operations = {
884 	.open		= partitions_open,
885 	.read		= seq_read,
886 	.llseek		= seq_lseek,
887 	.release	= seq_release,
888 };
889 #endif
890 
891 
892 static struct kobject *base_probe(dev_t devt, int *partno, void *data)
893 {
894 	if (request_module("block-major-%d-%d", MAJOR(devt), MINOR(devt)) > 0)
895 		/* Make old-style 2.4 aliases work */
896 		request_module("block-major-%d", MAJOR(devt));
897 	return NULL;
898 }
899 
900 static int __init genhd_device_init(void)
901 {
902 	int error;
903 
904 	block_class.dev_kobj = sysfs_dev_block_kobj;
905 	error = class_register(&block_class);
906 	if (unlikely(error))
907 		return error;
908 	bdev_map = kobj_map_init(base_probe, &block_class_lock);
909 	blk_dev_init();
910 
911 	register_blkdev(BLOCK_EXT_MAJOR, "blkext");
912 
913 	/* create top-level block dir */
914 	if (!sysfs_deprecated)
915 		block_depr = kobject_create_and_add("block", NULL);
916 	return 0;
917 }
918 
919 subsys_initcall(genhd_device_init);
920 
921 static ssize_t disk_range_show(struct device *dev,
922 			       struct device_attribute *attr, char *buf)
923 {
924 	struct gendisk *disk = dev_to_disk(dev);
925 
926 	return sprintf(buf, "%d\n", disk->minors);
927 }
928 
929 static ssize_t disk_ext_range_show(struct device *dev,
930 				   struct device_attribute *attr, char *buf)
931 {
932 	struct gendisk *disk = dev_to_disk(dev);
933 
934 	return sprintf(buf, "%d\n", disk_max_parts(disk));
935 }
936 
937 static ssize_t disk_removable_show(struct device *dev,
938 				   struct device_attribute *attr, char *buf)
939 {
940 	struct gendisk *disk = dev_to_disk(dev);
941 
942 	return sprintf(buf, "%d\n",
943 		       (disk->flags & GENHD_FL_REMOVABLE ? 1 : 0));
944 }
945 
946 static ssize_t disk_ro_show(struct device *dev,
947 				   struct device_attribute *attr, char *buf)
948 {
949 	struct gendisk *disk = dev_to_disk(dev);
950 
951 	return sprintf(buf, "%d\n", get_disk_ro(disk) ? 1 : 0);
952 }
953 
954 static ssize_t disk_capability_show(struct device *dev,
955 				    struct device_attribute *attr, char *buf)
956 {
957 	struct gendisk *disk = dev_to_disk(dev);
958 
959 	return sprintf(buf, "%x\n", disk->flags);
960 }
961 
962 static ssize_t disk_alignment_offset_show(struct device *dev,
963 					  struct device_attribute *attr,
964 					  char *buf)
965 {
966 	struct gendisk *disk = dev_to_disk(dev);
967 
968 	return sprintf(buf, "%d\n", queue_alignment_offset(disk->queue));
969 }
970 
971 static ssize_t disk_discard_alignment_show(struct device *dev,
972 					   struct device_attribute *attr,
973 					   char *buf)
974 {
975 	struct gendisk *disk = dev_to_disk(dev);
976 
977 	return sprintf(buf, "%d\n", queue_discard_alignment(disk->queue));
978 }
979 
980 static DEVICE_ATTR(range, S_IRUGO, disk_range_show, NULL);
981 static DEVICE_ATTR(ext_range, S_IRUGO, disk_ext_range_show, NULL);
982 static DEVICE_ATTR(removable, S_IRUGO, disk_removable_show, NULL);
983 static DEVICE_ATTR(ro, S_IRUGO, disk_ro_show, NULL);
984 static DEVICE_ATTR(size, S_IRUGO, part_size_show, NULL);
985 static DEVICE_ATTR(alignment_offset, S_IRUGO, disk_alignment_offset_show, NULL);
986 static DEVICE_ATTR(discard_alignment, S_IRUGO, disk_discard_alignment_show,
987 		   NULL);
988 static DEVICE_ATTR(capability, S_IRUGO, disk_capability_show, NULL);
989 static DEVICE_ATTR(stat, S_IRUGO, part_stat_show, NULL);
990 static DEVICE_ATTR(inflight, S_IRUGO, part_inflight_show, NULL);
991 #ifdef CONFIG_FAIL_MAKE_REQUEST
992 static struct device_attribute dev_attr_fail =
993 	__ATTR(make-it-fail, S_IRUGO|S_IWUSR, part_fail_show, part_fail_store);
994 #endif
995 #ifdef CONFIG_FAIL_IO_TIMEOUT
996 static struct device_attribute dev_attr_fail_timeout =
997 	__ATTR(io-timeout-fail,  S_IRUGO|S_IWUSR, part_timeout_show,
998 		part_timeout_store);
999 #endif
1000 
1001 static struct attribute *disk_attrs[] = {
1002 	&dev_attr_range.attr,
1003 	&dev_attr_ext_range.attr,
1004 	&dev_attr_removable.attr,
1005 	&dev_attr_ro.attr,
1006 	&dev_attr_size.attr,
1007 	&dev_attr_alignment_offset.attr,
1008 	&dev_attr_discard_alignment.attr,
1009 	&dev_attr_capability.attr,
1010 	&dev_attr_stat.attr,
1011 	&dev_attr_inflight.attr,
1012 #ifdef CONFIG_FAIL_MAKE_REQUEST
1013 	&dev_attr_fail.attr,
1014 #endif
1015 #ifdef CONFIG_FAIL_IO_TIMEOUT
1016 	&dev_attr_fail_timeout.attr,
1017 #endif
1018 	NULL
1019 };
1020 
1021 static struct attribute_group disk_attr_group = {
1022 	.attrs = disk_attrs,
1023 };
1024 
1025 static const struct attribute_group *disk_attr_groups[] = {
1026 	&disk_attr_group,
1027 	NULL
1028 };
1029 
1030 /**
1031  * disk_replace_part_tbl - replace disk->part_tbl in RCU-safe way
1032  * @disk: disk to replace part_tbl for
1033  * @new_ptbl: new part_tbl to install
1034  *
1035  * Replace disk->part_tbl with @new_ptbl in RCU-safe way.  The
1036  * original ptbl is freed using RCU callback.
1037  *
1038  * LOCKING:
1039  * Matching bd_mutx locked.
1040  */
1041 static void disk_replace_part_tbl(struct gendisk *disk,
1042 				  struct disk_part_tbl *new_ptbl)
1043 {
1044 	struct disk_part_tbl *old_ptbl = disk->part_tbl;
1045 
1046 	rcu_assign_pointer(disk->part_tbl, new_ptbl);
1047 
1048 	if (old_ptbl) {
1049 		rcu_assign_pointer(old_ptbl->last_lookup, NULL);
1050 		kfree_rcu(old_ptbl, rcu_head);
1051 	}
1052 }
1053 
1054 /**
1055  * disk_expand_part_tbl - expand disk->part_tbl
1056  * @disk: disk to expand part_tbl for
1057  * @partno: expand such that this partno can fit in
1058  *
1059  * Expand disk->part_tbl such that @partno can fit in.  disk->part_tbl
1060  * uses RCU to allow unlocked dereferencing for stats and other stuff.
1061  *
1062  * LOCKING:
1063  * Matching bd_mutex locked, might sleep.
1064  *
1065  * RETURNS:
1066  * 0 on success, -errno on failure.
1067  */
1068 int disk_expand_part_tbl(struct gendisk *disk, int partno)
1069 {
1070 	struct disk_part_tbl *old_ptbl = disk->part_tbl;
1071 	struct disk_part_tbl *new_ptbl;
1072 	int len = old_ptbl ? old_ptbl->len : 0;
1073 	int i, target;
1074 	size_t size;
1075 
1076 	/*
1077 	 * check for int overflow, since we can get here from blkpg_ioctl()
1078 	 * with a user passed 'partno'.
1079 	 */
1080 	target = partno + 1;
1081 	if (target < 0)
1082 		return -EINVAL;
1083 
1084 	/* disk_max_parts() is zero during initialization, ignore if so */
1085 	if (disk_max_parts(disk) && target > disk_max_parts(disk))
1086 		return -EINVAL;
1087 
1088 	if (target <= len)
1089 		return 0;
1090 
1091 	size = sizeof(*new_ptbl) + target * sizeof(new_ptbl->part[0]);
1092 	new_ptbl = kzalloc_node(size, GFP_KERNEL, disk->node_id);
1093 	if (!new_ptbl)
1094 		return -ENOMEM;
1095 
1096 	new_ptbl->len = target;
1097 
1098 	for (i = 0; i < len; i++)
1099 		rcu_assign_pointer(new_ptbl->part[i], old_ptbl->part[i]);
1100 
1101 	disk_replace_part_tbl(disk, new_ptbl);
1102 	return 0;
1103 }
1104 
1105 static void disk_release(struct device *dev)
1106 {
1107 	struct gendisk *disk = dev_to_disk(dev);
1108 
1109 	blk_free_devt(dev->devt);
1110 	disk_release_events(disk);
1111 	kfree(disk->random);
1112 	disk_replace_part_tbl(disk, NULL);
1113 	free_part_stats(&disk->part0);
1114 	free_part_info(&disk->part0);
1115 	if (disk->queue)
1116 		blk_put_queue(disk->queue);
1117 	kfree(disk);
1118 }
1119 struct class block_class = {
1120 	.name		= "block",
1121 };
1122 
1123 static char *block_devnode(struct device *dev, umode_t *mode,
1124 			   kuid_t *uid, kgid_t *gid)
1125 {
1126 	struct gendisk *disk = dev_to_disk(dev);
1127 
1128 	if (disk->devnode)
1129 		return disk->devnode(disk, mode);
1130 	return NULL;
1131 }
1132 
1133 static struct device_type disk_type = {
1134 	.name		= "disk",
1135 	.groups		= disk_attr_groups,
1136 	.release	= disk_release,
1137 	.devnode	= block_devnode,
1138 };
1139 
1140 #ifdef CONFIG_PROC_FS
1141 /*
1142  * aggregate disk stat collector.  Uses the same stats that the sysfs
1143  * entries do, above, but makes them available through one seq_file.
1144  *
1145  * The output looks suspiciously like /proc/partitions with a bunch of
1146  * extra fields.
1147  */
1148 static int diskstats_show(struct seq_file *seqf, void *v)
1149 {
1150 	struct gendisk *gp = v;
1151 	struct disk_part_iter piter;
1152 	struct hd_struct *hd;
1153 	char buf[BDEVNAME_SIZE];
1154 	int cpu;
1155 
1156 	/*
1157 	if (&disk_to_dev(gp)->kobj.entry == block_class.devices.next)
1158 		seq_puts(seqf,	"major minor name"
1159 				"     rio rmerge rsect ruse wio wmerge "
1160 				"wsect wuse running use aveq"
1161 				"\n\n");
1162 	*/
1163 
1164 	disk_part_iter_init(&piter, gp, DISK_PITER_INCL_EMPTY_PART0);
1165 	while ((hd = disk_part_iter_next(&piter))) {
1166 		cpu = part_stat_lock();
1167 		part_round_stats(cpu, hd);
1168 		part_stat_unlock();
1169 		seq_printf(seqf, "%4d %7d %s %lu %lu %lu "
1170 			   "%u %lu %lu %lu %u %u %u %u\n",
1171 			   MAJOR(part_devt(hd)), MINOR(part_devt(hd)),
1172 			   disk_name(gp, hd->partno, buf),
1173 			   part_stat_read(hd, ios[READ]),
1174 			   part_stat_read(hd, merges[READ]),
1175 			   part_stat_read(hd, sectors[READ]),
1176 			   jiffies_to_msecs(part_stat_read(hd, ticks[READ])),
1177 			   part_stat_read(hd, ios[WRITE]),
1178 			   part_stat_read(hd, merges[WRITE]),
1179 			   part_stat_read(hd, sectors[WRITE]),
1180 			   jiffies_to_msecs(part_stat_read(hd, ticks[WRITE])),
1181 			   part_in_flight(hd),
1182 			   jiffies_to_msecs(part_stat_read(hd, io_ticks)),
1183 			   jiffies_to_msecs(part_stat_read(hd, time_in_queue))
1184 			);
1185 	}
1186 	disk_part_iter_exit(&piter);
1187 
1188 	return 0;
1189 }
1190 
1191 static const struct seq_operations diskstats_op = {
1192 	.start	= disk_seqf_start,
1193 	.next	= disk_seqf_next,
1194 	.stop	= disk_seqf_stop,
1195 	.show	= diskstats_show
1196 };
1197 
1198 static int diskstats_open(struct inode *inode, struct file *file)
1199 {
1200 	return seq_open(file, &diskstats_op);
1201 }
1202 
1203 static const struct file_operations proc_diskstats_operations = {
1204 	.open		= diskstats_open,
1205 	.read		= seq_read,
1206 	.llseek		= seq_lseek,
1207 	.release	= seq_release,
1208 };
1209 
1210 static int __init proc_genhd_init(void)
1211 {
1212 	proc_create("diskstats", 0, NULL, &proc_diskstats_operations);
1213 	proc_create("partitions", 0, NULL, &proc_partitions_operations);
1214 	return 0;
1215 }
1216 module_init(proc_genhd_init);
1217 #endif /* CONFIG_PROC_FS */
1218 
1219 dev_t blk_lookup_devt(const char *name, int partno)
1220 {
1221 	dev_t devt = MKDEV(0, 0);
1222 	struct class_dev_iter iter;
1223 	struct device *dev;
1224 
1225 	class_dev_iter_init(&iter, &block_class, NULL, &disk_type);
1226 	while ((dev = class_dev_iter_next(&iter))) {
1227 		struct gendisk *disk = dev_to_disk(dev);
1228 		struct hd_struct *part;
1229 
1230 		if (strcmp(dev_name(dev), name))
1231 			continue;
1232 
1233 		if (partno < disk->minors) {
1234 			/* We need to return the right devno, even
1235 			 * if the partition doesn't exist yet.
1236 			 */
1237 			devt = MKDEV(MAJOR(dev->devt),
1238 				     MINOR(dev->devt) + partno);
1239 			break;
1240 		}
1241 		part = disk_get_part(disk, partno);
1242 		if (part) {
1243 			devt = part_devt(part);
1244 			disk_put_part(part);
1245 			break;
1246 		}
1247 		disk_put_part(part);
1248 	}
1249 	class_dev_iter_exit(&iter);
1250 	return devt;
1251 }
1252 EXPORT_SYMBOL(blk_lookup_devt);
1253 
1254 struct gendisk *alloc_disk(int minors)
1255 {
1256 	return alloc_disk_node(minors, NUMA_NO_NODE);
1257 }
1258 EXPORT_SYMBOL(alloc_disk);
1259 
1260 struct gendisk *alloc_disk_node(int minors, int node_id)
1261 {
1262 	struct gendisk *disk;
1263 
1264 	disk = kzalloc_node(sizeof(struct gendisk), GFP_KERNEL, node_id);
1265 	if (disk) {
1266 		if (!init_part_stats(&disk->part0)) {
1267 			kfree(disk);
1268 			return NULL;
1269 		}
1270 		disk->node_id = node_id;
1271 		if (disk_expand_part_tbl(disk, 0)) {
1272 			free_part_stats(&disk->part0);
1273 			kfree(disk);
1274 			return NULL;
1275 		}
1276 		disk->part_tbl->part[0] = &disk->part0;
1277 
1278 		/*
1279 		 * set_capacity() and get_capacity() currently don't use
1280 		 * seqcounter to read/update the part0->nr_sects. Still init
1281 		 * the counter as we can read the sectors in IO submission
1282 		 * patch using seqence counters.
1283 		 *
1284 		 * TODO: Ideally set_capacity() and get_capacity() should be
1285 		 * converted to make use of bd_mutex and sequence counters.
1286 		 */
1287 		seqcount_init(&disk->part0.nr_sects_seq);
1288 		hd_ref_init(&disk->part0);
1289 
1290 		disk->minors = minors;
1291 		rand_initialize_disk(disk);
1292 		disk_to_dev(disk)->class = &block_class;
1293 		disk_to_dev(disk)->type = &disk_type;
1294 		device_initialize(disk_to_dev(disk));
1295 	}
1296 	return disk;
1297 }
1298 EXPORT_SYMBOL(alloc_disk_node);
1299 
1300 struct kobject *get_disk(struct gendisk *disk)
1301 {
1302 	struct module *owner;
1303 	struct kobject *kobj;
1304 
1305 	if (!disk->fops)
1306 		return NULL;
1307 	owner = disk->fops->owner;
1308 	if (owner && !try_module_get(owner))
1309 		return NULL;
1310 	kobj = kobject_get(&disk_to_dev(disk)->kobj);
1311 	if (kobj == NULL) {
1312 		module_put(owner);
1313 		return NULL;
1314 	}
1315 	return kobj;
1316 
1317 }
1318 
1319 EXPORT_SYMBOL(get_disk);
1320 
1321 void put_disk(struct gendisk *disk)
1322 {
1323 	if (disk)
1324 		kobject_put(&disk_to_dev(disk)->kobj);
1325 }
1326 
1327 EXPORT_SYMBOL(put_disk);
1328 
1329 static void set_disk_ro_uevent(struct gendisk *gd, int ro)
1330 {
1331 	char event[] = "DISK_RO=1";
1332 	char *envp[] = { event, NULL };
1333 
1334 	if (!ro)
1335 		event[8] = '0';
1336 	kobject_uevent_env(&disk_to_dev(gd)->kobj, KOBJ_CHANGE, envp);
1337 }
1338 
1339 void set_device_ro(struct block_device *bdev, int flag)
1340 {
1341 	bdev->bd_part->policy = flag;
1342 }
1343 
1344 EXPORT_SYMBOL(set_device_ro);
1345 
1346 void set_disk_ro(struct gendisk *disk, int flag)
1347 {
1348 	struct disk_part_iter piter;
1349 	struct hd_struct *part;
1350 
1351 	if (disk->part0.policy != flag) {
1352 		set_disk_ro_uevent(disk, flag);
1353 		disk->part0.policy = flag;
1354 	}
1355 
1356 	disk_part_iter_init(&piter, disk, DISK_PITER_INCL_EMPTY);
1357 	while ((part = disk_part_iter_next(&piter)))
1358 		part->policy = flag;
1359 	disk_part_iter_exit(&piter);
1360 }
1361 
1362 EXPORT_SYMBOL(set_disk_ro);
1363 
1364 int bdev_read_only(struct block_device *bdev)
1365 {
1366 	if (!bdev)
1367 		return 0;
1368 	return bdev->bd_part->policy;
1369 }
1370 
1371 EXPORT_SYMBOL(bdev_read_only);
1372 
1373 int invalidate_partition(struct gendisk *disk, int partno)
1374 {
1375 	int res = 0;
1376 	struct block_device *bdev = bdget_disk(disk, partno);
1377 	if (bdev) {
1378 		fsync_bdev(bdev);
1379 		res = __invalidate_device(bdev, true);
1380 		bdput(bdev);
1381 	}
1382 	return res;
1383 }
1384 
1385 EXPORT_SYMBOL(invalidate_partition);
1386 
1387 /*
1388  * Disk events - monitor disk events like media change and eject request.
1389  */
1390 struct disk_events {
1391 	struct list_head	node;		/* all disk_event's */
1392 	struct gendisk		*disk;		/* the associated disk */
1393 	spinlock_t		lock;
1394 
1395 	struct mutex		block_mutex;	/* protects blocking */
1396 	int			block;		/* event blocking depth */
1397 	unsigned int		pending;	/* events already sent out */
1398 	unsigned int		clearing;	/* events being cleared */
1399 
1400 	long			poll_msecs;	/* interval, -1 for default */
1401 	struct delayed_work	dwork;
1402 };
1403 
1404 static const char *disk_events_strs[] = {
1405 	[ilog2(DISK_EVENT_MEDIA_CHANGE)]	= "media_change",
1406 	[ilog2(DISK_EVENT_EJECT_REQUEST)]	= "eject_request",
1407 };
1408 
1409 static char *disk_uevents[] = {
1410 	[ilog2(DISK_EVENT_MEDIA_CHANGE)]	= "DISK_MEDIA_CHANGE=1",
1411 	[ilog2(DISK_EVENT_EJECT_REQUEST)]	= "DISK_EJECT_REQUEST=1",
1412 };
1413 
1414 /* list of all disk_events */
1415 static DEFINE_MUTEX(disk_events_mutex);
1416 static LIST_HEAD(disk_events);
1417 
1418 /* disable in-kernel polling by default */
1419 static unsigned long disk_events_dfl_poll_msecs	= 0;
1420 
1421 static unsigned long disk_events_poll_jiffies(struct gendisk *disk)
1422 {
1423 	struct disk_events *ev = disk->ev;
1424 	long intv_msecs = 0;
1425 
1426 	/*
1427 	 * If device-specific poll interval is set, always use it.  If
1428 	 * the default is being used, poll iff there are events which
1429 	 * can't be monitored asynchronously.
1430 	 */
1431 	if (ev->poll_msecs >= 0)
1432 		intv_msecs = ev->poll_msecs;
1433 	else if (disk->events & ~disk->async_events)
1434 		intv_msecs = disk_events_dfl_poll_msecs;
1435 
1436 	return msecs_to_jiffies(intv_msecs);
1437 }
1438 
1439 /**
1440  * disk_block_events - block and flush disk event checking
1441  * @disk: disk to block events for
1442  *
1443  * On return from this function, it is guaranteed that event checking
1444  * isn't in progress and won't happen until unblocked by
1445  * disk_unblock_events().  Events blocking is counted and the actual
1446  * unblocking happens after the matching number of unblocks are done.
1447  *
1448  * Note that this intentionally does not block event checking from
1449  * disk_clear_events().
1450  *
1451  * CONTEXT:
1452  * Might sleep.
1453  */
1454 void disk_block_events(struct gendisk *disk)
1455 {
1456 	struct disk_events *ev = disk->ev;
1457 	unsigned long flags;
1458 	bool cancel;
1459 
1460 	if (!ev)
1461 		return;
1462 
1463 	/*
1464 	 * Outer mutex ensures that the first blocker completes canceling
1465 	 * the event work before further blockers are allowed to finish.
1466 	 */
1467 	mutex_lock(&ev->block_mutex);
1468 
1469 	spin_lock_irqsave(&ev->lock, flags);
1470 	cancel = !ev->block++;
1471 	spin_unlock_irqrestore(&ev->lock, flags);
1472 
1473 	if (cancel)
1474 		cancel_delayed_work_sync(&disk->ev->dwork);
1475 
1476 	mutex_unlock(&ev->block_mutex);
1477 }
1478 
1479 static void __disk_unblock_events(struct gendisk *disk, bool check_now)
1480 {
1481 	struct disk_events *ev = disk->ev;
1482 	unsigned long intv;
1483 	unsigned long flags;
1484 
1485 	spin_lock_irqsave(&ev->lock, flags);
1486 
1487 	if (WARN_ON_ONCE(ev->block <= 0))
1488 		goto out_unlock;
1489 
1490 	if (--ev->block)
1491 		goto out_unlock;
1492 
1493 	/*
1494 	 * Not exactly a latency critical operation, set poll timer
1495 	 * slack to 25% and kick event check.
1496 	 */
1497 	intv = disk_events_poll_jiffies(disk);
1498 	set_timer_slack(&ev->dwork.timer, intv / 4);
1499 	if (check_now)
1500 		queue_delayed_work(system_freezable_power_efficient_wq,
1501 				&ev->dwork, 0);
1502 	else if (intv)
1503 		queue_delayed_work(system_freezable_power_efficient_wq,
1504 				&ev->dwork, intv);
1505 out_unlock:
1506 	spin_unlock_irqrestore(&ev->lock, flags);
1507 }
1508 
1509 /**
1510  * disk_unblock_events - unblock disk event checking
1511  * @disk: disk to unblock events for
1512  *
1513  * Undo disk_block_events().  When the block count reaches zero, it
1514  * starts events polling if configured.
1515  *
1516  * CONTEXT:
1517  * Don't care.  Safe to call from irq context.
1518  */
1519 void disk_unblock_events(struct gendisk *disk)
1520 {
1521 	if (disk->ev)
1522 		__disk_unblock_events(disk, false);
1523 }
1524 
1525 /**
1526  * disk_flush_events - schedule immediate event checking and flushing
1527  * @disk: disk to check and flush events for
1528  * @mask: events to flush
1529  *
1530  * Schedule immediate event checking on @disk if not blocked.  Events in
1531  * @mask are scheduled to be cleared from the driver.  Note that this
1532  * doesn't clear the events from @disk->ev.
1533  *
1534  * CONTEXT:
1535  * If @mask is non-zero must be called with bdev->bd_mutex held.
1536  */
1537 void disk_flush_events(struct gendisk *disk, unsigned int mask)
1538 {
1539 	struct disk_events *ev = disk->ev;
1540 
1541 	if (!ev)
1542 		return;
1543 
1544 	spin_lock_irq(&ev->lock);
1545 	ev->clearing |= mask;
1546 	if (!ev->block)
1547 		mod_delayed_work(system_freezable_power_efficient_wq,
1548 				&ev->dwork, 0);
1549 	spin_unlock_irq(&ev->lock);
1550 }
1551 
1552 /**
1553  * disk_clear_events - synchronously check, clear and return pending events
1554  * @disk: disk to fetch and clear events from
1555  * @mask: mask of events to be fetched and cleared
1556  *
1557  * Disk events are synchronously checked and pending events in @mask
1558  * are cleared and returned.  This ignores the block count.
1559  *
1560  * CONTEXT:
1561  * Might sleep.
1562  */
1563 unsigned int disk_clear_events(struct gendisk *disk, unsigned int mask)
1564 {
1565 	const struct block_device_operations *bdops = disk->fops;
1566 	struct disk_events *ev = disk->ev;
1567 	unsigned int pending;
1568 	unsigned int clearing = mask;
1569 
1570 	if (!ev) {
1571 		/* for drivers still using the old ->media_changed method */
1572 		if ((mask & DISK_EVENT_MEDIA_CHANGE) &&
1573 		    bdops->media_changed && bdops->media_changed(disk))
1574 			return DISK_EVENT_MEDIA_CHANGE;
1575 		return 0;
1576 	}
1577 
1578 	disk_block_events(disk);
1579 
1580 	/*
1581 	 * store the union of mask and ev->clearing on the stack so that the
1582 	 * race with disk_flush_events does not cause ambiguity (ev->clearing
1583 	 * can still be modified even if events are blocked).
1584 	 */
1585 	spin_lock_irq(&ev->lock);
1586 	clearing |= ev->clearing;
1587 	ev->clearing = 0;
1588 	spin_unlock_irq(&ev->lock);
1589 
1590 	disk_check_events(ev, &clearing);
1591 	/*
1592 	 * if ev->clearing is not 0, the disk_flush_events got called in the
1593 	 * middle of this function, so we want to run the workfn without delay.
1594 	 */
1595 	__disk_unblock_events(disk, ev->clearing ? true : false);
1596 
1597 	/* then, fetch and clear pending events */
1598 	spin_lock_irq(&ev->lock);
1599 	pending = ev->pending & mask;
1600 	ev->pending &= ~mask;
1601 	spin_unlock_irq(&ev->lock);
1602 	WARN_ON_ONCE(clearing & mask);
1603 
1604 	return pending;
1605 }
1606 
1607 /*
1608  * Separate this part out so that a different pointer for clearing_ptr can be
1609  * passed in for disk_clear_events.
1610  */
1611 static void disk_events_workfn(struct work_struct *work)
1612 {
1613 	struct delayed_work *dwork = to_delayed_work(work);
1614 	struct disk_events *ev = container_of(dwork, struct disk_events, dwork);
1615 
1616 	disk_check_events(ev, &ev->clearing);
1617 }
1618 
1619 static void disk_check_events(struct disk_events *ev,
1620 			      unsigned int *clearing_ptr)
1621 {
1622 	struct gendisk *disk = ev->disk;
1623 	char *envp[ARRAY_SIZE(disk_uevents) + 1] = { };
1624 	unsigned int clearing = *clearing_ptr;
1625 	unsigned int events;
1626 	unsigned long intv;
1627 	int nr_events = 0, i;
1628 
1629 	/* check events */
1630 	events = disk->fops->check_events(disk, clearing);
1631 
1632 	/* accumulate pending events and schedule next poll if necessary */
1633 	spin_lock_irq(&ev->lock);
1634 
1635 	events &= ~ev->pending;
1636 	ev->pending |= events;
1637 	*clearing_ptr &= ~clearing;
1638 
1639 	intv = disk_events_poll_jiffies(disk);
1640 	if (!ev->block && intv)
1641 		queue_delayed_work(system_freezable_power_efficient_wq,
1642 				&ev->dwork, intv);
1643 
1644 	spin_unlock_irq(&ev->lock);
1645 
1646 	/*
1647 	 * Tell userland about new events.  Only the events listed in
1648 	 * @disk->events are reported.  Unlisted events are processed the
1649 	 * same internally but never get reported to userland.
1650 	 */
1651 	for (i = 0; i < ARRAY_SIZE(disk_uevents); i++)
1652 		if (events & disk->events & (1 << i))
1653 			envp[nr_events++] = disk_uevents[i];
1654 
1655 	if (nr_events)
1656 		kobject_uevent_env(&disk_to_dev(disk)->kobj, KOBJ_CHANGE, envp);
1657 }
1658 
1659 /*
1660  * A disk events enabled device has the following sysfs nodes under
1661  * its /sys/block/X/ directory.
1662  *
1663  * events		: list of all supported events
1664  * events_async		: list of events which can be detected w/o polling
1665  * events_poll_msecs	: polling interval, 0: disable, -1: system default
1666  */
1667 static ssize_t __disk_events_show(unsigned int events, char *buf)
1668 {
1669 	const char *delim = "";
1670 	ssize_t pos = 0;
1671 	int i;
1672 
1673 	for (i = 0; i < ARRAY_SIZE(disk_events_strs); i++)
1674 		if (events & (1 << i)) {
1675 			pos += sprintf(buf + pos, "%s%s",
1676 				       delim, disk_events_strs[i]);
1677 			delim = " ";
1678 		}
1679 	if (pos)
1680 		pos += sprintf(buf + pos, "\n");
1681 	return pos;
1682 }
1683 
1684 static ssize_t disk_events_show(struct device *dev,
1685 				struct device_attribute *attr, char *buf)
1686 {
1687 	struct gendisk *disk = dev_to_disk(dev);
1688 
1689 	return __disk_events_show(disk->events, buf);
1690 }
1691 
1692 static ssize_t disk_events_async_show(struct device *dev,
1693 				      struct device_attribute *attr, char *buf)
1694 {
1695 	struct gendisk *disk = dev_to_disk(dev);
1696 
1697 	return __disk_events_show(disk->async_events, buf);
1698 }
1699 
1700 static ssize_t disk_events_poll_msecs_show(struct device *dev,
1701 					   struct device_attribute *attr,
1702 					   char *buf)
1703 {
1704 	struct gendisk *disk = dev_to_disk(dev);
1705 
1706 	return sprintf(buf, "%ld\n", disk->ev->poll_msecs);
1707 }
1708 
1709 static ssize_t disk_events_poll_msecs_store(struct device *dev,
1710 					    struct device_attribute *attr,
1711 					    const char *buf, size_t count)
1712 {
1713 	struct gendisk *disk = dev_to_disk(dev);
1714 	long intv;
1715 
1716 	if (!count || !sscanf(buf, "%ld", &intv))
1717 		return -EINVAL;
1718 
1719 	if (intv < 0 && intv != -1)
1720 		return -EINVAL;
1721 
1722 	disk_block_events(disk);
1723 	disk->ev->poll_msecs = intv;
1724 	__disk_unblock_events(disk, true);
1725 
1726 	return count;
1727 }
1728 
1729 static const DEVICE_ATTR(events, S_IRUGO, disk_events_show, NULL);
1730 static const DEVICE_ATTR(events_async, S_IRUGO, disk_events_async_show, NULL);
1731 static const DEVICE_ATTR(events_poll_msecs, S_IRUGO|S_IWUSR,
1732 			 disk_events_poll_msecs_show,
1733 			 disk_events_poll_msecs_store);
1734 
1735 static const struct attribute *disk_events_attrs[] = {
1736 	&dev_attr_events.attr,
1737 	&dev_attr_events_async.attr,
1738 	&dev_attr_events_poll_msecs.attr,
1739 	NULL,
1740 };
1741 
1742 /*
1743  * The default polling interval can be specified by the kernel
1744  * parameter block.events_dfl_poll_msecs which defaults to 0
1745  * (disable).  This can also be modified runtime by writing to
1746  * /sys/module/block/events_dfl_poll_msecs.
1747  */
1748 static int disk_events_set_dfl_poll_msecs(const char *val,
1749 					  const struct kernel_param *kp)
1750 {
1751 	struct disk_events *ev;
1752 	int ret;
1753 
1754 	ret = param_set_ulong(val, kp);
1755 	if (ret < 0)
1756 		return ret;
1757 
1758 	mutex_lock(&disk_events_mutex);
1759 
1760 	list_for_each_entry(ev, &disk_events, node)
1761 		disk_flush_events(ev->disk, 0);
1762 
1763 	mutex_unlock(&disk_events_mutex);
1764 
1765 	return 0;
1766 }
1767 
1768 static const struct kernel_param_ops disk_events_dfl_poll_msecs_param_ops = {
1769 	.set	= disk_events_set_dfl_poll_msecs,
1770 	.get	= param_get_ulong,
1771 };
1772 
1773 #undef MODULE_PARAM_PREFIX
1774 #define MODULE_PARAM_PREFIX	"block."
1775 
1776 module_param_cb(events_dfl_poll_msecs, &disk_events_dfl_poll_msecs_param_ops,
1777 		&disk_events_dfl_poll_msecs, 0644);
1778 
1779 /*
1780  * disk_{alloc|add|del|release}_events - initialize and destroy disk_events.
1781  */
1782 static void disk_alloc_events(struct gendisk *disk)
1783 {
1784 	struct disk_events *ev;
1785 
1786 	if (!disk->fops->check_events)
1787 		return;
1788 
1789 	ev = kzalloc(sizeof(*ev), GFP_KERNEL);
1790 	if (!ev) {
1791 		pr_warn("%s: failed to initialize events\n", disk->disk_name);
1792 		return;
1793 	}
1794 
1795 	INIT_LIST_HEAD(&ev->node);
1796 	ev->disk = disk;
1797 	spin_lock_init(&ev->lock);
1798 	mutex_init(&ev->block_mutex);
1799 	ev->block = 1;
1800 	ev->poll_msecs = -1;
1801 	INIT_DELAYED_WORK(&ev->dwork, disk_events_workfn);
1802 
1803 	disk->ev = ev;
1804 }
1805 
1806 static void disk_add_events(struct gendisk *disk)
1807 {
1808 	if (!disk->ev)
1809 		return;
1810 
1811 	/* FIXME: error handling */
1812 	if (sysfs_create_files(&disk_to_dev(disk)->kobj, disk_events_attrs) < 0)
1813 		pr_warn("%s: failed to create sysfs files for events\n",
1814 			disk->disk_name);
1815 
1816 	mutex_lock(&disk_events_mutex);
1817 	list_add_tail(&disk->ev->node, &disk_events);
1818 	mutex_unlock(&disk_events_mutex);
1819 
1820 	/*
1821 	 * Block count is initialized to 1 and the following initial
1822 	 * unblock kicks it into action.
1823 	 */
1824 	__disk_unblock_events(disk, true);
1825 }
1826 
1827 static void disk_del_events(struct gendisk *disk)
1828 {
1829 	if (!disk->ev)
1830 		return;
1831 
1832 	disk_block_events(disk);
1833 
1834 	mutex_lock(&disk_events_mutex);
1835 	list_del_init(&disk->ev->node);
1836 	mutex_unlock(&disk_events_mutex);
1837 
1838 	sysfs_remove_files(&disk_to_dev(disk)->kobj, disk_events_attrs);
1839 }
1840 
1841 static void disk_release_events(struct gendisk *disk)
1842 {
1843 	/* the block count should be 1 from disk_del_events() */
1844 	WARN_ON_ONCE(disk->ev && disk->ev->block != 1);
1845 	kfree(disk->ev);
1846 }
1847