330 				struct child_latency_info *lat_info,  in scale_cookie_change()  argument
335 	unsigned long old = atomic_read(&lat_info->scale_cookie);  in scale_cookie_change()
344 			atomic_set(&lat_info->scale_cookie,  in scale_cookie_change()
347 			atomic_inc(&lat_info->scale_cookie);  in scale_cookie_change()
349 			atomic_add(scale, &lat_info->scale_cookie);  in scale_cookie_change()
359 				atomic_dec(&lat_info->scale_cookie);  in scale_cookie_change()
361 			atomic_sub(scale, &lat_info->scale_cookie);  in scale_cookie_change()
402 	struct child_latency_info *lat_info;  in check_scale_change()  local
412 	lat_info = &parent->child_lat;  in check_scale_change()
413 	cur_cookie = atomic_read(&lat_info->scale_cookie);  in check_scale_change()
414 	scale_lat = READ_ONCE(lat_info->scale_lat);  in check_scale_change()
440 		samples_thresh = lat_info->nr_samples * 5;  in check_scale_change()
527 	struct child_latency_info *lat_info;  in iolatency_check_latencies()  local
546 	lat_info = &parent->child_lat;  in iolatency_check_latencies()
552 	    atomic_read(&lat_info->scale_cookie) == DEFAULT_SCALE_COOKIE)  in iolatency_check_latencies()
556 	spin_lock_irqsave(&lat_info->lock, flags);  in iolatency_check_latencies()
559 	lat_info->nr_samples -= iolat->nr_samples;  in iolatency_check_latencies()
560 	lat_info->nr_samples += latency_stat_samples(iolat, &iolat->cur_stat);  in iolatency_check_latencies()
563 	if ((lat_info->last_scale_event >= now ||  in iolatency_check_latencies()
564 	    now - lat_info->last_scale_event < BLKIOLATENCY_MIN_ADJUST_TIME))  in iolatency_check_latencies()
572 		if (lat_info->scale_grp == iolat) {  in iolatency_check_latencies()
573 			lat_info->last_scale_event = now;  in iolatency_check_latencies()
574 			scale_cookie_change(iolat->blkiolat, lat_info, true);  in iolatency_check_latencies()
576 	} else if (lat_info->scale_lat == 0 ||  in iolatency_check_latencies()
577 		   lat_info->scale_lat >= iolat->min_lat_nsec) {  in iolatency_check_latencies()
578 		lat_info->last_scale_event = now;  in iolatency_check_latencies()
579 		if (!lat_info->scale_grp ||  in iolatency_check_latencies()
580 		    lat_info->scale_lat > iolat->min_lat_nsec) {  in iolatency_check_latencies()
581 			WRITE_ONCE(lat_info->scale_lat, iolat->min_lat_nsec);  in iolatency_check_latencies()
582 			lat_info->scale_grp = iolat;  in iolatency_check_latencies()
584 		scale_cookie_change(iolat->blkiolat, lat_info, false);  in iolatency_check_latencies()
588 	spin_unlock_irqrestore(&lat_info->lock, flags);  in iolatency_check_latencies()
670 		struct child_latency_info *lat_info;  in blkiolatency_timer_fn()  local
685 		lat_info = &iolat->child_lat;  in blkiolatency_timer_fn()
686 		cookie = atomic_read(&lat_info->scale_cookie);  in blkiolatency_timer_fn()
691 		spin_lock_irqsave(&lat_info->lock, flags);  in blkiolatency_timer_fn()
692 		if (lat_info->last_scale_event >= now)  in blkiolatency_timer_fn()
699 		if (lat_info->scale_grp == NULL) {  in blkiolatency_timer_fn()
700 			scale_cookie_change(iolat->blkiolat, lat_info, true);  in blkiolatency_timer_fn()
709 		if (now - lat_info->last_scale_event >=  in blkiolatency_timer_fn()
711 			lat_info->scale_grp = NULL;  in blkiolatency_timer_fn()
713 		spin_unlock_irqrestore(&lat_info->lock, flags);  in blkiolatency_timer_fn()
813 		struct child_latency_info *lat_info;  in iolatency_clear_scaling()  local
817 		lat_info = &iolat->child_lat;  in iolatency_clear_scaling()
818 		spin_lock(&lat_info->lock);  in iolatency_clear_scaling()
819 		atomic_set(&lat_info->scale_cookie, DEFAULT_SCALE_COOKIE);  in iolatency_clear_scaling()
820 		lat_info->last_scale_event = 0;  in iolatency_clear_scaling()
821 		lat_info->scale_grp = NULL;  in iolatency_clear_scaling()
822 		lat_info->scale_lat = 0;  in iolatency_clear_scaling()
823 		spin_unlock(&lat_info->lock);  in iolatency_clear_scaling()