314 static void blk_trace_free(struct request_queue *q, struct blk_trace *bt)  in blk_trace_free()  argument
323 		debugfs_lookup_and_remove("dropped", q->debugfs_dir);  in blk_trace_free()
324 		debugfs_lookup_and_remove("msg", q->debugfs_dir);  in blk_trace_free()
380 static void blk_trace_cleanup(struct request_queue *q, struct blk_trace *bt)  in blk_trace_cleanup()  argument
384 	blk_trace_free(q, bt);  in blk_trace_cleanup()
388 static int __blk_trace_remove(struct request_queue *q)  in __blk_trace_remove()  argument
392 	bt = rcu_replace_pointer(q->blk_trace, NULL,  in __blk_trace_remove()
393 				 lockdep_is_held(&q->debugfs_mutex));  in __blk_trace_remove()
397 	blk_trace_cleanup(q, bt);  in __blk_trace_remove()
402 int blk_trace_remove(struct request_queue *q)  in blk_trace_remove()  argument
406 	mutex_lock(&q->debugfs_mutex);  in blk_trace_remove()
407 	ret = __blk_trace_remove(q);  in blk_trace_remove()
408 	mutex_unlock(&q->debugfs_mutex);  in blk_trace_remove()
514 static int do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev,  in do_blk_trace_setup()  argument
522 	lockdep_assert_held(&q->debugfs_mutex);  in do_blk_trace_setup()
540 	if (rcu_dereference_protected(q->blk_trace,  in do_blk_trace_setup()
541 				      lockdep_is_held(&q->debugfs_mutex))) {  in do_blk_trace_setup()
567 		dir = q->debugfs_dir;  in do_blk_trace_setup()
611 	rcu_assign_pointer(q->blk_trace, bt);  in do_blk_trace_setup()
617 		blk_trace_free(q, bt);  in do_blk_trace_setup()
621 static int __blk_trace_setup(struct request_queue *q, char *name, dev_t dev,  in __blk_trace_setup()  argument
631 	ret = do_blk_trace_setup(q, name, dev, bdev, &buts);  in __blk_trace_setup()
636 		__blk_trace_remove(q);  in __blk_trace_setup()
642 int blk_trace_setup(struct request_queue *q, char *name, dev_t dev,  in blk_trace_setup()  argument
648 	mutex_lock(&q->debugfs_mutex);  in blk_trace_setup()
649 	ret = __blk_trace_setup(q, name, dev, bdev, arg);  in blk_trace_setup()
650 	mutex_unlock(&q->debugfs_mutex);  in blk_trace_setup()
657 static int compat_blk_trace_setup(struct request_queue *q, char *name,  in compat_blk_trace_setup()  argument
677 	ret = do_blk_trace_setup(q, name, dev, bdev, &buts);  in compat_blk_trace_setup()
682 		__blk_trace_remove(q);  in compat_blk_trace_setup()
690 static int __blk_trace_startstop(struct request_queue *q, int start)  in __blk_trace_startstop()  argument
694 	bt = rcu_dereference_protected(q->blk_trace,  in __blk_trace_startstop()
695 				       lockdep_is_held(&q->debugfs_mutex));  in __blk_trace_startstop()
705 int blk_trace_startstop(struct request_queue *q, int start)  in blk_trace_startstop()  argument
709 	mutex_lock(&q->debugfs_mutex);  in blk_trace_startstop()
710 	ret = __blk_trace_startstop(q, start);  in blk_trace_startstop()
711 	mutex_unlock(&q->debugfs_mutex);  in blk_trace_startstop()
732 	struct request_queue *q = bdev_get_queue(bdev);  in blk_trace_ioctl()  local
736 	mutex_lock(&q->debugfs_mutex);  in blk_trace_ioctl()
741 		ret = __blk_trace_setup(q, b, bdev->bd_dev, bdev, arg);  in blk_trace_ioctl()
746 		ret = compat_blk_trace_setup(q, b, bdev->bd_dev, bdev, arg);  in blk_trace_ioctl()
753 		ret = __blk_trace_startstop(q, start);  in blk_trace_ioctl()
756 		ret = __blk_trace_remove(q);  in blk_trace_ioctl()
763 	mutex_unlock(&q->debugfs_mutex);  in blk_trace_ioctl()
772 void blk_trace_shutdown(struct request_queue *q)  in blk_trace_shutdown()  argument
774 	if (rcu_dereference_protected(q->blk_trace,  in blk_trace_shutdown()
775 				      lockdep_is_held(&q->debugfs_mutex)))  in blk_trace_shutdown()
776 		__blk_trace_remove(q);  in blk_trace_shutdown()
780 static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio)  in blk_trace_bio_get_cgid()  argument
786 	bt = rcu_dereference_protected(q->blk_trace, 1);  in blk_trace_bio_get_cgid()
796 static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio)  in blk_trace_bio_get_cgid()  argument
808 	return blk_trace_bio_get_cgid(rq->q, rq->bio);  in blk_trace_request_get_cgid()
833 	bt = rcu_dereference(rq->q->blk_trace);  in blk_add_trace_rq()
891 static void blk_add_trace_bio(struct request_queue *q, struct bio *bio,  in blk_add_trace_bio()  argument
897 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_bio()
905 			blk_trace_bio_get_cgid(q, bio));  in blk_add_trace_bio()
915 				       struct request_queue *q, struct bio *bio)  in blk_add_trace_bio_complete()  argument
917 	blk_add_trace_bio(q, bio, BLK_TA_COMPLETE,  in blk_add_trace_bio_complete()
943 static void blk_add_trace_plug(void *ignore, struct request_queue *q)  in blk_add_trace_plug()  argument
948 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_plug()
954 static void blk_add_trace_unplug(void *ignore, struct request_queue *q,  in blk_add_trace_unplug()  argument
960 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_unplug()
977 	struct request_queue *q = bio->bi_bdev->bd_disk->queue;  in blk_add_trace_split()  local
981 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_split()
989 				blk_trace_bio_get_cgid(q, bio));  in blk_add_trace_split()
1006 	struct request_queue *q = bio->bi_bdev->bd_disk->queue;  in blk_add_trace_bio_remap()  local
1011 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_bio_remap()
1024 			sizeof(r), &r, blk_trace_bio_get_cgid(q, bio));  in blk_add_trace_bio_remap()
1047 	bt = rcu_dereference(rq->q->blk_trace);  in blk_add_trace_rq_remap()
1054 	r.device_to   = cpu_to_be32(disk_devt(rq->q->disk));  in blk_add_trace_rq_remap()
1078 	bt = rcu_dereference(rq->q->blk_trace);  in blk_add_driver_data()
1607 static int blk_trace_remove_queue(struct request_queue *q)  in blk_trace_remove_queue()  argument
1611 	bt = rcu_replace_pointer(q->blk_trace, NULL,  in blk_trace_remove_queue()
1612 				 lockdep_is_held(&q->debugfs_mutex));  in blk_trace_remove_queue()
1620 	blk_trace_free(q, bt);  in blk_trace_remove_queue()
1627 static int blk_trace_setup_queue(struct request_queue *q,  in blk_trace_setup_queue()  argument
1646 	rcu_assign_pointer(q->blk_trace, bt);  in blk_trace_setup_queue()
1651 	blk_trace_free(q, bt);  in blk_trace_setup_queue()
1768 	struct request_queue *q = bdev_get_queue(bdev);  in sysfs_blk_trace_attr_show()  local
1772 	mutex_lock(&q->debugfs_mutex);  in sysfs_blk_trace_attr_show()
1774 	bt = rcu_dereference_protected(q->blk_trace,  in sysfs_blk_trace_attr_show()
1775 				       lockdep_is_held(&q->debugfs_mutex));  in sysfs_blk_trace_attr_show()
1793 	mutex_unlock(&q->debugfs_mutex);  in sysfs_blk_trace_attr_show()
1802 	struct request_queue *q = bdev_get_queue(bdev);  in sysfs_blk_trace_attr_store()  local
1823 	mutex_lock(&q->debugfs_mutex);  in sysfs_blk_trace_attr_store()
1825 	bt = rcu_dereference_protected(q->blk_trace,  in sysfs_blk_trace_attr_store()
1826 				       lockdep_is_held(&q->debugfs_mutex));  in sysfs_blk_trace_attr_store()
1833 			ret = blk_trace_setup_queue(q, bdev);  in sysfs_blk_trace_attr_store()
1835 			ret = blk_trace_remove_queue(q);  in sysfs_blk_trace_attr_store()
1841 		ret = blk_trace_setup_queue(q, bdev);  in sysfs_blk_trace_attr_store()
1842 		bt = rcu_dereference_protected(q->blk_trace,  in sysfs_blk_trace_attr_store()
1843 				lockdep_is_held(&q->debugfs_mutex));  in sysfs_blk_trace_attr_store()
1858 	mutex_unlock(&q->debugfs_mutex);  in sysfs_blk_trace_attr_store()