59 static int dmz_reclaim_align_wp(struct dmz_reclaim *zrc, struct dm_zone *zone,  in dmz_reclaim_align_wp()  argument
62 	struct dmz_metadata *zmd = zrc->metadata;  in dmz_reclaim_align_wp()
102 	struct dmz_reclaim *zrc = context;  in dmz_reclaim_kcopy_end()  local
105 		zrc->kc_err = -EIO;  in dmz_reclaim_kcopy_end()
107 		zrc->kc_err = 0;  in dmz_reclaim_kcopy_end()
109 	clear_bit_unlock(DMZ_RECLAIM_KCOPY, &zrc->flags);  in dmz_reclaim_kcopy_end()
111 	wake_up_bit(&zrc->flags, DMZ_RECLAIM_KCOPY);  in dmz_reclaim_kcopy_end()
117 static int dmz_reclaim_copy(struct dmz_reclaim *zrc,  in dmz_reclaim_copy()  argument
120 	struct dmz_metadata *zmd = zrc->metadata;  in dmz_reclaim_copy()
160 			ret = dmz_reclaim_align_wp(zrc, dst_zone, block);  in dmz_reclaim_copy()
174 		set_bit(DMZ_RECLAIM_KCOPY, &zrc->flags);  in dmz_reclaim_copy()
175 		dm_kcopyd_copy(zrc->kc, &src, 1, &dst, flags,  in dmz_reclaim_copy()
176 			       dmz_reclaim_kcopy_end, zrc);  in dmz_reclaim_copy()
179 		wait_on_bit_io(&zrc->flags, DMZ_RECLAIM_KCOPY,  in dmz_reclaim_copy()
181 		if (zrc->kc_err)  in dmz_reclaim_copy()
182 			return zrc->kc_err;  in dmz_reclaim_copy()
196 static int dmz_reclaim_buf(struct dmz_reclaim *zrc, struct dm_zone *dzone)  in dmz_reclaim_buf()  argument
200 	struct dmz_metadata *zmd = zrc->metadata;  in dmz_reclaim_buf()
204 		dmz_metadata_label(zmd), zrc->dev_idx,  in dmz_reclaim_buf()
209 	ret = dmz_reclaim_copy(zrc, bzone, dzone);  in dmz_reclaim_buf()
235 static int dmz_reclaim_seq_data(struct dmz_reclaim *zrc, struct dm_zone *dzone)  in dmz_reclaim_seq_data()  argument
239 	struct dmz_metadata *zmd = zrc->metadata;  in dmz_reclaim_seq_data()
243 		dmz_metadata_label(zmd), zrc->dev_idx,  in dmz_reclaim_seq_data()
248 	ret = dmz_reclaim_copy(zrc, dzone, bzone);  in dmz_reclaim_seq_data()
280 static int dmz_reclaim_rnd_data(struct dmz_reclaim *zrc, struct dm_zone *dzone)  in dmz_reclaim_rnd_data()  argument
284 	struct dmz_metadata *zmd = zrc->metadata;  in dmz_reclaim_rnd_data()
291 	szone = dmz_alloc_zone(zmd, zrc->dev_idx,  in dmz_reclaim_rnd_data()
302 		dmz_metadata_label(zmd), zrc->dev_idx, chunk,  in dmz_reclaim_rnd_data()
308 	ret = dmz_reclaim_copy(zrc, dzone, szone);  in dmz_reclaim_rnd_data()
340 static void dmz_reclaim_empty(struct dmz_reclaim *zrc, struct dm_zone *dzone)  in dmz_reclaim_empty()  argument
342 	struct dmz_metadata *zmd = zrc->metadata;  in dmz_reclaim_empty()
356 static inline int dmz_target_idle(struct dmz_reclaim *zrc)  in dmz_target_idle()  argument
358 	return time_is_before_jiffies(zrc->atime + DMZ_IDLE_PERIOD);  in dmz_target_idle()
364 static int dmz_do_reclaim(struct dmz_reclaim *zrc)  in dmz_do_reclaim()  argument
366 	struct dmz_metadata *zmd = zrc->metadata;  in dmz_do_reclaim()
373 	dzone = dmz_get_zone_for_reclaim(zmd, zrc->dev_idx,  in dmz_do_reclaim()
374 					 dmz_target_idle(zrc));  in dmz_do_reclaim()
377 			dmz_metadata_label(zmd), zrc->dev_idx);  in dmz_do_reclaim()
386 			dmz_reclaim_empty(zrc, dzone);  in dmz_do_reclaim()
393 			ret = dmz_reclaim_rnd_data(zrc, dzone);  in dmz_do_reclaim()
408 			ret = dmz_reclaim_buf(zrc, dzone);  in dmz_do_reclaim()
416 			ret = dmz_reclaim_seq_data(zrc, dzone);  in dmz_do_reclaim()
423 				dmz_metadata_label(zmd), zrc->dev_idx,  in dmz_do_reclaim()
427 				dmz_metadata_label(zmd), zrc->dev_idx,  in dmz_do_reclaim()
433 	ret = dmz_flush_metadata(zrc->metadata);  in dmz_do_reclaim()
436 			dmz_metadata_label(zmd), zrc->dev_idx, rzone->id, ret);  in dmz_do_reclaim()
441 		dmz_metadata_label(zmd), zrc->dev_idx,  in dmz_do_reclaim()
446 static unsigned int dmz_reclaim_percentage(struct dmz_reclaim *zrc)  in dmz_reclaim_percentage()  argument
448 	struct dmz_metadata *zmd = zrc->metadata;  in dmz_reclaim_percentage()
456 		nr_zones = dmz_nr_rnd_zones(zmd, zrc->dev_idx);  in dmz_reclaim_percentage()
457 		nr_unmap = dmz_nr_unmap_rnd_zones(zmd, zrc->dev_idx);  in dmz_reclaim_percentage()
467 static bool dmz_should_reclaim(struct dmz_reclaim *zrc, unsigned int p_unmap)  in dmz_should_reclaim()  argument
471 	nr_reclaim = dmz_nr_rnd_zones(zrc->metadata, zrc->dev_idx);  in dmz_should_reclaim()
473 	if (dmz_nr_cache_zones(zrc->metadata)) {  in dmz_should_reclaim()
479 		if (zrc->dev_idx == 0)  in dmz_should_reclaim()
481 		nr_reclaim += dmz_nr_cache_zones(zrc->metadata);  in dmz_should_reclaim()
485 	if (dmz_target_idle(zrc) && nr_reclaim)  in dmz_should_reclaim()
504 	struct dmz_reclaim *zrc = container_of(work, struct dmz_reclaim, work.work);  in dmz_reclaim_work()  local
505 	struct dmz_metadata *zmd = zrc->metadata;  in dmz_reclaim_work()
512 	p_unmap = dmz_reclaim_percentage(zrc);  in dmz_reclaim_work()
513 	if (!dmz_should_reclaim(zrc, p_unmap)) {  in dmz_reclaim_work()
514 		mod_delayed_work(zrc->wq, &zrc->work, DMZ_IDLE_PERIOD);  in dmz_reclaim_work()
524 	if (dmz_target_idle(zrc) || p_unmap < DMZ_RECLAIM_LOW_UNMAP_ZONES / 2) {  in dmz_reclaim_work()
526 		zrc->kc_throttle.throttle = 100;  in dmz_reclaim_work()
529 		zrc->kc_throttle.throttle = min(75U, 100U - p_unmap / 2);  in dmz_reclaim_work()
533 		dmz_metadata_label(zmd), zrc->dev_idx,  in dmz_reclaim_work()
534 		zrc->kc_throttle.throttle,  in dmz_reclaim_work()
535 		(dmz_target_idle(zrc) ? "Idle" : "Busy"),  in dmz_reclaim_work()
538 		dmz_nr_unmap_rnd_zones(zmd, zrc->dev_idx),  in dmz_reclaim_work()
539 		dmz_nr_rnd_zones(zmd, zrc->dev_idx));  in dmz_reclaim_work()
541 	ret = dmz_do_reclaim(zrc);  in dmz_reclaim_work()
547 	dmz_schedule_reclaim(zrc);  in dmz_reclaim_work()
556 	struct dmz_reclaim *zrc;  in dmz_ctr_reclaim()  local
559 	zrc = kzalloc(sizeof(struct dmz_reclaim), GFP_KERNEL);  in dmz_ctr_reclaim()
560 	if (!zrc)  in dmz_ctr_reclaim()
563 	zrc->metadata = zmd;  in dmz_ctr_reclaim()
564 	zrc->atime = jiffies;  in dmz_ctr_reclaim()
565 	zrc->dev_idx = idx;  in dmz_ctr_reclaim()
568 	zrc->kc = dm_kcopyd_client_create(&zrc->kc_throttle);  in dmz_ctr_reclaim()
569 	if (IS_ERR(zrc->kc)) {  in dmz_ctr_reclaim()
570 		ret = PTR_ERR(zrc->kc);  in dmz_ctr_reclaim()
571 		zrc->kc = NULL;  in dmz_ctr_reclaim()
576 	INIT_DELAYED_WORK(&zrc->work, dmz_reclaim_work);  in dmz_ctr_reclaim()
577 	zrc->wq = alloc_ordered_workqueue("dmz_rwq_%s_%d", WQ_MEM_RECLAIM,  in dmz_ctr_reclaim()
579 	if (!zrc->wq) {  in dmz_ctr_reclaim()
584 	*reclaim = zrc;  in dmz_ctr_reclaim()
585 	queue_delayed_work(zrc->wq, &zrc->work, 0);  in dmz_ctr_reclaim()
589 	if (zrc->kc)  in dmz_ctr_reclaim()
590 		dm_kcopyd_client_destroy(zrc->kc);  in dmz_ctr_reclaim()
591 	kfree(zrc);  in dmz_ctr_reclaim()
599 void dmz_dtr_reclaim(struct dmz_reclaim *zrc)  in dmz_dtr_reclaim()  argument
601 	cancel_delayed_work_sync(&zrc->work);  in dmz_dtr_reclaim()
602 	destroy_workqueue(zrc->wq);  in dmz_dtr_reclaim()
603 	dm_kcopyd_client_destroy(zrc->kc);  in dmz_dtr_reclaim()
604 	kfree(zrc);  in dmz_dtr_reclaim()
610 void dmz_suspend_reclaim(struct dmz_reclaim *zrc)  in dmz_suspend_reclaim()  argument
612 	cancel_delayed_work_sync(&zrc->work);  in dmz_suspend_reclaim()
618 void dmz_resume_reclaim(struct dmz_reclaim *zrc)  in dmz_resume_reclaim()  argument
620 	queue_delayed_work(zrc->wq, &zrc->work, DMZ_IDLE_PERIOD);  in dmz_resume_reclaim()
626 void dmz_reclaim_bio_acc(struct dmz_reclaim *zrc)  in dmz_reclaim_bio_acc()  argument
628 	zrc->atime = jiffies;  in dmz_reclaim_bio_acc()
634 void dmz_schedule_reclaim(struct dmz_reclaim *zrc)  in dmz_schedule_reclaim()  argument
636 	unsigned int p_unmap = dmz_reclaim_percentage(zrc);  in dmz_schedule_reclaim()
638 	if (dmz_should_reclaim(zrc, p_unmap))  in dmz_schedule_reclaim()
639 		mod_delayed_work(zrc->wq, &zrc->work, 0);  in dmz_schedule_reclaim()