xref: /openbmc/linux/drivers/md/dm-stats.c (revision d3aa3e06)
1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/errno.h>
3 #include <linux/numa.h>
4 #include <linux/slab.h>
5 #include <linux/rculist.h>
6 #include <linux/threads.h>
7 #include <linux/preempt.h>
8 #include <linux/irqflags.h>
9 #include <linux/vmalloc.h>
10 #include <linux/mm.h>
11 #include <linux/module.h>
12 #include <linux/device-mapper.h>
13 
14 #include "dm-core.h"
15 #include "dm-stats.h"
16 
17 #define DM_MSG_PREFIX "stats"
18 
19 static int dm_stat_need_rcu_barrier;
20 
21 /*
22  * Using 64-bit values to avoid overflow (which is a
23  * problem that block/genhd.c's IO accounting has).
24  */
25 struct dm_stat_percpu {
26 	unsigned long long sectors[2];
27 	unsigned long long ios[2];
28 	unsigned long long merges[2];
29 	unsigned long long ticks[2];
30 	unsigned long long io_ticks[2];
31 	unsigned long long io_ticks_total;
32 	unsigned long long time_in_queue;
33 	unsigned long long *histogram;
34 };
35 
36 struct dm_stat_shared {
37 	atomic_t in_flight[2];
38 	unsigned long long stamp;
39 	struct dm_stat_percpu tmp;
40 };
41 
42 struct dm_stat {
43 	struct list_head list_entry;
44 	int id;
45 	unsigned int stat_flags;
46 	size_t n_entries;
47 	sector_t start;
48 	sector_t end;
49 	sector_t step;
50 	unsigned int n_histogram_entries;
51 	unsigned long long *histogram_boundaries;
52 	const char *program_id;
53 	const char *aux_data;
54 	struct rcu_head rcu_head;
55 	size_t shared_alloc_size;
56 	size_t percpu_alloc_size;
57 	size_t histogram_alloc_size;
58 	struct dm_stat_percpu *stat_percpu[NR_CPUS];
59 	struct dm_stat_shared stat_shared[];
60 };
61 
62 #define STAT_PRECISE_TIMESTAMPS		1
63 
64 struct dm_stats_last_position {
65 	sector_t last_sector;
66 	unsigned int last_rw;
67 };
68 
69 /*
70  * A typo on the command line could possibly make the kernel run out of memory
71  * and crash. To prevent the crash we account all used memory. We fail if we
72  * exhaust 1/4 of all memory or 1/2 of vmalloc space.
73  */
74 #define DM_STATS_MEMORY_FACTOR		4
75 #define DM_STATS_VMALLOC_FACTOR		2
76 
77 static DEFINE_SPINLOCK(shared_memory_lock);
78 
79 static unsigned long shared_memory_amount;
80 
__check_shared_memory(size_t alloc_size)81 static bool __check_shared_memory(size_t alloc_size)
82 {
83 	size_t a;
84 
85 	a = shared_memory_amount + alloc_size;
86 	if (a < shared_memory_amount)
87 		return false;
88 	if (a >> PAGE_SHIFT > totalram_pages() / DM_STATS_MEMORY_FACTOR)
89 		return false;
90 #ifdef CONFIG_MMU
91 	if (a > (VMALLOC_END - VMALLOC_START) / DM_STATS_VMALLOC_FACTOR)
92 		return false;
93 #endif
94 	return true;
95 }
96 
check_shared_memory(size_t alloc_size)97 static bool check_shared_memory(size_t alloc_size)
98 {
99 	bool ret;
100 
101 	spin_lock_irq(&shared_memory_lock);
102 
103 	ret = __check_shared_memory(alloc_size);
104 
105 	spin_unlock_irq(&shared_memory_lock);
106 
107 	return ret;
108 }
109 
claim_shared_memory(size_t alloc_size)110 static bool claim_shared_memory(size_t alloc_size)
111 {
112 	spin_lock_irq(&shared_memory_lock);
113 
114 	if (!__check_shared_memory(alloc_size)) {
115 		spin_unlock_irq(&shared_memory_lock);
116 		return false;
117 	}
118 
119 	shared_memory_amount += alloc_size;
120 
121 	spin_unlock_irq(&shared_memory_lock);
122 
123 	return true;
124 }
125 
free_shared_memory(size_t alloc_size)126 static void free_shared_memory(size_t alloc_size)
127 {
128 	unsigned long flags;
129 
130 	spin_lock_irqsave(&shared_memory_lock, flags);
131 
132 	if (WARN_ON_ONCE(shared_memory_amount < alloc_size)) {
133 		spin_unlock_irqrestore(&shared_memory_lock, flags);
134 		DMCRIT("Memory usage accounting bug.");
135 		return;
136 	}
137 
138 	shared_memory_amount -= alloc_size;
139 
140 	spin_unlock_irqrestore(&shared_memory_lock, flags);
141 }
142 
dm_kvzalloc(size_t alloc_size,int node)143 static void *dm_kvzalloc(size_t alloc_size, int node)
144 {
145 	void *p;
146 
147 	if (!claim_shared_memory(alloc_size))
148 		return NULL;
149 
150 	p = kvzalloc_node(alloc_size, GFP_KERNEL | __GFP_NOMEMALLOC, node);
151 	if (p)
152 		return p;
153 
154 	free_shared_memory(alloc_size);
155 
156 	return NULL;
157 }
158 
dm_kvfree(void * ptr,size_t alloc_size)159 static void dm_kvfree(void *ptr, size_t alloc_size)
160 {
161 	if (!ptr)
162 		return;
163 
164 	free_shared_memory(alloc_size);
165 
166 	kvfree(ptr);
167 }
168 
dm_stat_free(struct rcu_head * head)169 static void dm_stat_free(struct rcu_head *head)
170 {
171 	int cpu;
172 	struct dm_stat *s = container_of(head, struct dm_stat, rcu_head);
173 
174 	kfree(s->histogram_boundaries);
175 	kfree(s->program_id);
176 	kfree(s->aux_data);
177 	for_each_possible_cpu(cpu) {
178 		dm_kvfree(s->stat_percpu[cpu][0].histogram, s->histogram_alloc_size);
179 		dm_kvfree(s->stat_percpu[cpu], s->percpu_alloc_size);
180 	}
181 	dm_kvfree(s->stat_shared[0].tmp.histogram, s->histogram_alloc_size);
182 	dm_kvfree(s, s->shared_alloc_size);
183 }
184 
dm_stat_in_flight(struct dm_stat_shared * shared)185 static int dm_stat_in_flight(struct dm_stat_shared *shared)
186 {
187 	return atomic_read(&shared->in_flight[READ]) +
188 	       atomic_read(&shared->in_flight[WRITE]);
189 }
190 
dm_stats_init(struct dm_stats * stats)191 int dm_stats_init(struct dm_stats *stats)
192 {
193 	int cpu;
194 	struct dm_stats_last_position *last;
195 
196 	mutex_init(&stats->mutex);
197 	INIT_LIST_HEAD(&stats->list);
198 	stats->precise_timestamps = false;
199 	stats->last = alloc_percpu(struct dm_stats_last_position);
200 	if (!stats->last)
201 		return -ENOMEM;
202 
203 	for_each_possible_cpu(cpu) {
204 		last = per_cpu_ptr(stats->last, cpu);
205 		last->last_sector = (sector_t)ULLONG_MAX;
206 		last->last_rw = UINT_MAX;
207 	}
208 
209 	return 0;
210 }
211 
dm_stats_cleanup(struct dm_stats * stats)212 void dm_stats_cleanup(struct dm_stats *stats)
213 {
214 	size_t ni;
215 	struct dm_stat *s;
216 	struct dm_stat_shared *shared;
217 
218 	while (!list_empty(&stats->list)) {
219 		s = container_of(stats->list.next, struct dm_stat, list_entry);
220 		list_del(&s->list_entry);
221 		for (ni = 0; ni < s->n_entries; ni++) {
222 			shared = &s->stat_shared[ni];
223 			if (WARN_ON(dm_stat_in_flight(shared))) {
224 				DMCRIT("leaked in-flight counter at index %lu "
225 				       "(start %llu, end %llu, step %llu): reads %d, writes %d",
226 				       (unsigned long)ni,
227 				       (unsigned long long)s->start,
228 				       (unsigned long long)s->end,
229 				       (unsigned long long)s->step,
230 				       atomic_read(&shared->in_flight[READ]),
231 				       atomic_read(&shared->in_flight[WRITE]));
232 			}
233 			cond_resched();
234 		}
235 		dm_stat_free(&s->rcu_head);
236 	}
237 	free_percpu(stats->last);
238 	mutex_destroy(&stats->mutex);
239 }
240 
dm_stats_recalc_precise_timestamps(struct dm_stats * stats)241 static void dm_stats_recalc_precise_timestamps(struct dm_stats *stats)
242 {
243 	struct list_head *l;
244 	struct dm_stat *tmp_s;
245 	bool precise_timestamps = false;
246 
247 	list_for_each(l, &stats->list) {
248 		tmp_s = container_of(l, struct dm_stat, list_entry);
249 		if (tmp_s->stat_flags & STAT_PRECISE_TIMESTAMPS) {
250 			precise_timestamps = true;
251 			break;
252 		}
253 	}
254 	stats->precise_timestamps = precise_timestamps;
255 }
256 
dm_stats_create(struct dm_stats * stats,sector_t start,sector_t end,sector_t step,unsigned int stat_flags,unsigned int n_histogram_entries,unsigned long long * histogram_boundaries,const char * program_id,const char * aux_data,void (* suspend_callback)(struct mapped_device *),void (* resume_callback)(struct mapped_device *),struct mapped_device * md)257 static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end,
258 			   sector_t step, unsigned int stat_flags,
259 			   unsigned int n_histogram_entries,
260 			   unsigned long long *histogram_boundaries,
261 			   const char *program_id, const char *aux_data,
262 			   void (*suspend_callback)(struct mapped_device *),
263 			   void (*resume_callback)(struct mapped_device *),
264 			   struct mapped_device *md)
265 {
266 	struct list_head *l;
267 	struct dm_stat *s, *tmp_s;
268 	sector_t n_entries;
269 	size_t ni;
270 	size_t shared_alloc_size;
271 	size_t percpu_alloc_size;
272 	size_t histogram_alloc_size;
273 	struct dm_stat_percpu *p;
274 	int cpu;
275 	int ret_id;
276 	int r;
277 
278 	if (end < start || !step)
279 		return -EINVAL;
280 
281 	n_entries = end - start;
282 	if (dm_sector_div64(n_entries, step))
283 		n_entries++;
284 
285 	if (n_entries != (size_t)n_entries || !(size_t)(n_entries + 1))
286 		return -EOVERFLOW;
287 
288 	shared_alloc_size = struct_size(s, stat_shared, n_entries);
289 	if ((shared_alloc_size - sizeof(struct dm_stat)) / sizeof(struct dm_stat_shared) != n_entries)
290 		return -EOVERFLOW;
291 
292 	percpu_alloc_size = (size_t)n_entries * sizeof(struct dm_stat_percpu);
293 	if (percpu_alloc_size / sizeof(struct dm_stat_percpu) != n_entries)
294 		return -EOVERFLOW;
295 
296 	histogram_alloc_size = (n_histogram_entries + 1) * (size_t)n_entries * sizeof(unsigned long long);
297 	if (histogram_alloc_size / (n_histogram_entries + 1) != (size_t)n_entries * sizeof(unsigned long long))
298 		return -EOVERFLOW;
299 
300 	if (!check_shared_memory(shared_alloc_size + histogram_alloc_size +
301 				 num_possible_cpus() * (percpu_alloc_size + histogram_alloc_size)))
302 		return -ENOMEM;
303 
304 	s = dm_kvzalloc(shared_alloc_size, NUMA_NO_NODE);
305 	if (!s)
306 		return -ENOMEM;
307 
308 	s->stat_flags = stat_flags;
309 	s->n_entries = n_entries;
310 	s->start = start;
311 	s->end = end;
312 	s->step = step;
313 	s->shared_alloc_size = shared_alloc_size;
314 	s->percpu_alloc_size = percpu_alloc_size;
315 	s->histogram_alloc_size = histogram_alloc_size;
316 
317 	s->n_histogram_entries = n_histogram_entries;
318 	s->histogram_boundaries = kmemdup(histogram_boundaries,
319 					  s->n_histogram_entries * sizeof(unsigned long long), GFP_KERNEL);
320 	if (!s->histogram_boundaries) {
321 		r = -ENOMEM;
322 		goto out;
323 	}
324 
325 	s->program_id = kstrdup(program_id, GFP_KERNEL);
326 	if (!s->program_id) {
327 		r = -ENOMEM;
328 		goto out;
329 	}
330 	s->aux_data = kstrdup(aux_data, GFP_KERNEL);
331 	if (!s->aux_data) {
332 		r = -ENOMEM;
333 		goto out;
334 	}
335 
336 	for (ni = 0; ni < n_entries; ni++) {
337 		atomic_set(&s->stat_shared[ni].in_flight[READ], 0);
338 		atomic_set(&s->stat_shared[ni].in_flight[WRITE], 0);
339 		cond_resched();
340 	}
341 
342 	if (s->n_histogram_entries) {
343 		unsigned long long *hi;
344 
345 		hi = dm_kvzalloc(s->histogram_alloc_size, NUMA_NO_NODE);
346 		if (!hi) {
347 			r = -ENOMEM;
348 			goto out;
349 		}
350 		for (ni = 0; ni < n_entries; ni++) {
351 			s->stat_shared[ni].tmp.histogram = hi;
352 			hi += s->n_histogram_entries + 1;
353 			cond_resched();
354 		}
355 	}
356 
357 	for_each_possible_cpu(cpu) {
358 		p = dm_kvzalloc(percpu_alloc_size, cpu_to_node(cpu));
359 		if (!p) {
360 			r = -ENOMEM;
361 			goto out;
362 		}
363 		s->stat_percpu[cpu] = p;
364 		if (s->n_histogram_entries) {
365 			unsigned long long *hi;
366 
367 			hi = dm_kvzalloc(s->histogram_alloc_size, cpu_to_node(cpu));
368 			if (!hi) {
369 				r = -ENOMEM;
370 				goto out;
371 			}
372 			for (ni = 0; ni < n_entries; ni++) {
373 				p[ni].histogram = hi;
374 				hi += s->n_histogram_entries + 1;
375 				cond_resched();
376 			}
377 		}
378 	}
379 
380 	/*
381 	 * Suspend/resume to make sure there is no i/o in flight,
382 	 * so that newly created statistics will be exact.
383 	 *
384 	 * (note: we couldn't suspend earlier because we must not
385 	 * allocate memory while suspended)
386 	 */
387 	suspend_callback(md);
388 
389 	mutex_lock(&stats->mutex);
390 	s->id = 0;
391 	list_for_each(l, &stats->list) {
392 		tmp_s = container_of(l, struct dm_stat, list_entry);
393 		if (WARN_ON(tmp_s->id < s->id)) {
394 			r = -EINVAL;
395 			goto out_unlock_resume;
396 		}
397 		if (tmp_s->id > s->id)
398 			break;
399 		if (unlikely(s->id == INT_MAX)) {
400 			r = -ENFILE;
401 			goto out_unlock_resume;
402 		}
403 		s->id++;
404 	}
405 	ret_id = s->id;
406 	list_add_tail_rcu(&s->list_entry, l);
407 
408 	dm_stats_recalc_precise_timestamps(stats);
409 
410 	if (!static_key_enabled(&stats_enabled.key))
411 		static_branch_enable(&stats_enabled);
412 
413 	mutex_unlock(&stats->mutex);
414 
415 	resume_callback(md);
416 
417 	return ret_id;
418 
419 out_unlock_resume:
420 	mutex_unlock(&stats->mutex);
421 	resume_callback(md);
422 out:
423 	dm_stat_free(&s->rcu_head);
424 	return r;
425 }
426 
__dm_stats_find(struct dm_stats * stats,int id)427 static struct dm_stat *__dm_stats_find(struct dm_stats *stats, int id)
428 {
429 	struct dm_stat *s;
430 
431 	list_for_each_entry(s, &stats->list, list_entry) {
432 		if (s->id > id)
433 			break;
434 		if (s->id == id)
435 			return s;
436 	}
437 
438 	return NULL;
439 }
440 
dm_stats_delete(struct dm_stats * stats,int id)441 static int dm_stats_delete(struct dm_stats *stats, int id)
442 {
443 	struct dm_stat *s;
444 	int cpu;
445 
446 	mutex_lock(&stats->mutex);
447 
448 	s = __dm_stats_find(stats, id);
449 	if (!s) {
450 		mutex_unlock(&stats->mutex);
451 		return -ENOENT;
452 	}
453 
454 	list_del_rcu(&s->list_entry);
455 
456 	dm_stats_recalc_precise_timestamps(stats);
457 
458 	mutex_unlock(&stats->mutex);
459 
460 	/*
461 	 * vfree can't be called from RCU callback
462 	 */
463 	for_each_possible_cpu(cpu)
464 		if (is_vmalloc_addr(s->stat_percpu) ||
465 		    is_vmalloc_addr(s->stat_percpu[cpu][0].histogram))
466 			goto do_sync_free;
467 	if (is_vmalloc_addr(s) ||
468 	    is_vmalloc_addr(s->stat_shared[0].tmp.histogram)) {
469 do_sync_free:
470 		synchronize_rcu_expedited();
471 		dm_stat_free(&s->rcu_head);
472 	} else {
473 		WRITE_ONCE(dm_stat_need_rcu_barrier, 1);
474 		call_rcu(&s->rcu_head, dm_stat_free);
475 	}
476 	return 0;
477 }
478 
dm_stats_list(struct dm_stats * stats,const char * program,char * result,unsigned int maxlen)479 static int dm_stats_list(struct dm_stats *stats, const char *program,
480 			 char *result, unsigned int maxlen)
481 {
482 	struct dm_stat *s;
483 	sector_t len;
484 	unsigned int sz = 0;
485 
486 	/*
487 	 * Output format:
488 	 *   <region_id>: <start_sector>+<length> <step> <program_id> <aux_data>
489 	 */
490 
491 	mutex_lock(&stats->mutex);
492 	list_for_each_entry(s, &stats->list, list_entry) {
493 		if (!program || !strcmp(program, s->program_id)) {
494 			len = s->end - s->start;
495 			DMEMIT("%d: %llu+%llu %llu %s %s", s->id,
496 				(unsigned long long)s->start,
497 				(unsigned long long)len,
498 				(unsigned long long)s->step,
499 				s->program_id,
500 				s->aux_data);
501 			if (s->stat_flags & STAT_PRECISE_TIMESTAMPS)
502 				DMEMIT(" precise_timestamps");
503 			if (s->n_histogram_entries) {
504 				unsigned int i;
505 
506 				DMEMIT(" histogram:");
507 				for (i = 0; i < s->n_histogram_entries; i++) {
508 					if (i)
509 						DMEMIT(",");
510 					DMEMIT("%llu", s->histogram_boundaries[i]);
511 				}
512 			}
513 			DMEMIT("\n");
514 		}
515 		cond_resched();
516 	}
517 	mutex_unlock(&stats->mutex);
518 
519 	return 1;
520 }
521 
dm_stat_round(struct dm_stat * s,struct dm_stat_shared * shared,struct dm_stat_percpu * p)522 static void dm_stat_round(struct dm_stat *s, struct dm_stat_shared *shared,
523 			  struct dm_stat_percpu *p)
524 {
525 	/*
526 	 * This is racy, but so is part_round_stats_single.
527 	 */
528 	unsigned long long now, difference;
529 	unsigned int in_flight_read, in_flight_write;
530 
531 	if (likely(!(s->stat_flags & STAT_PRECISE_TIMESTAMPS)))
532 		now = jiffies;
533 	else
534 		now = ktime_to_ns(ktime_get());
535 
536 	difference = now - shared->stamp;
537 	if (!difference)
538 		return;
539 
540 	in_flight_read = (unsigned int)atomic_read(&shared->in_flight[READ]);
541 	in_flight_write = (unsigned int)atomic_read(&shared->in_flight[WRITE]);
542 	if (in_flight_read)
543 		p->io_ticks[READ] += difference;
544 	if (in_flight_write)
545 		p->io_ticks[WRITE] += difference;
546 	if (in_flight_read + in_flight_write) {
547 		p->io_ticks_total += difference;
548 		p->time_in_queue += (in_flight_read + in_flight_write) * difference;
549 	}
550 	shared->stamp = now;
551 }
552 
dm_stat_for_entry(struct dm_stat * s,size_t entry,int idx,sector_t len,struct dm_stats_aux * stats_aux,bool end,unsigned long duration_jiffies)553 static void dm_stat_for_entry(struct dm_stat *s, size_t entry,
554 			      int idx, sector_t len,
555 			      struct dm_stats_aux *stats_aux, bool end,
556 			      unsigned long duration_jiffies)
557 {
558 	struct dm_stat_shared *shared = &s->stat_shared[entry];
559 	struct dm_stat_percpu *p;
560 
561 	/*
562 	 * For strict correctness we should use local_irq_save/restore
563 	 * instead of preempt_disable/enable.
564 	 *
565 	 * preempt_disable/enable is racy if the driver finishes bios
566 	 * from non-interrupt context as well as from interrupt context
567 	 * or from more different interrupts.
568 	 *
569 	 * On 64-bit architectures the race only results in not counting some
570 	 * events, so it is acceptable.  On 32-bit architectures the race could
571 	 * cause the counter going off by 2^32, so we need to do proper locking
572 	 * there.
573 	 *
574 	 * part_stat_lock()/part_stat_unlock() have this race too.
575 	 */
576 #if BITS_PER_LONG == 32
577 	unsigned long flags;
578 
579 	local_irq_save(flags);
580 #else
581 	preempt_disable();
582 #endif
583 	p = &s->stat_percpu[smp_processor_id()][entry];
584 
585 	if (!end) {
586 		dm_stat_round(s, shared, p);
587 		atomic_inc(&shared->in_flight[idx]);
588 	} else {
589 		unsigned long long duration;
590 
591 		dm_stat_round(s, shared, p);
592 		atomic_dec(&shared->in_flight[idx]);
593 		p->sectors[idx] += len;
594 		p->ios[idx] += 1;
595 		p->merges[idx] += stats_aux->merged;
596 		if (!(s->stat_flags & STAT_PRECISE_TIMESTAMPS)) {
597 			p->ticks[idx] += duration_jiffies;
598 			duration = jiffies_to_msecs(duration_jiffies);
599 		} else {
600 			p->ticks[idx] += stats_aux->duration_ns;
601 			duration = stats_aux->duration_ns;
602 		}
603 		if (s->n_histogram_entries) {
604 			unsigned int lo = 0, hi = s->n_histogram_entries + 1;
605 
606 			while (lo + 1 < hi) {
607 				unsigned int mid = (lo + hi) / 2;
608 
609 				if (s->histogram_boundaries[mid - 1] > duration)
610 					hi = mid;
611 				else
612 					lo = mid;
613 			}
614 			p->histogram[lo]++;
615 		}
616 	}
617 
618 #if BITS_PER_LONG == 32
619 	local_irq_restore(flags);
620 #else
621 	preempt_enable();
622 #endif
623 }
624 
__dm_stat_bio(struct dm_stat * s,int bi_rw,sector_t bi_sector,sector_t end_sector,bool end,unsigned long duration_jiffies,struct dm_stats_aux * stats_aux)625 static void __dm_stat_bio(struct dm_stat *s, int bi_rw,
626 			  sector_t bi_sector, sector_t end_sector,
627 			  bool end, unsigned long duration_jiffies,
628 			  struct dm_stats_aux *stats_aux)
629 {
630 	sector_t rel_sector, offset, todo, fragment_len;
631 	size_t entry;
632 
633 	if (end_sector <= s->start || bi_sector >= s->end)
634 		return;
635 	if (unlikely(bi_sector < s->start)) {
636 		rel_sector = 0;
637 		todo = end_sector - s->start;
638 	} else {
639 		rel_sector = bi_sector - s->start;
640 		todo = end_sector - bi_sector;
641 	}
642 	if (unlikely(end_sector > s->end))
643 		todo -= (end_sector - s->end);
644 
645 	offset = dm_sector_div64(rel_sector, s->step);
646 	entry = rel_sector;
647 	do {
648 		if (WARN_ON_ONCE(entry >= s->n_entries)) {
649 			DMCRIT("Invalid area access in region id %d", s->id);
650 			return;
651 		}
652 		fragment_len = todo;
653 		if (fragment_len > s->step - offset)
654 			fragment_len = s->step - offset;
655 		dm_stat_for_entry(s, entry, bi_rw, fragment_len,
656 				  stats_aux, end, duration_jiffies);
657 		todo -= fragment_len;
658 		entry++;
659 		offset = 0;
660 	} while (unlikely(todo != 0));
661 }
662 
dm_stats_account_io(struct dm_stats * stats,unsigned long bi_rw,sector_t bi_sector,unsigned int bi_sectors,bool end,unsigned long start_time,struct dm_stats_aux * stats_aux)663 void dm_stats_account_io(struct dm_stats *stats, unsigned long bi_rw,
664 			 sector_t bi_sector, unsigned int bi_sectors, bool end,
665 			 unsigned long start_time,
666 			 struct dm_stats_aux *stats_aux)
667 {
668 	struct dm_stat *s;
669 	sector_t end_sector;
670 	struct dm_stats_last_position *last;
671 	bool got_precise_time;
672 	unsigned long duration_jiffies = 0;
673 
674 	if (unlikely(!bi_sectors))
675 		return;
676 
677 	end_sector = bi_sector + bi_sectors;
678 
679 	if (!end) {
680 		/*
681 		 * A race condition can at worst result in the merged flag being
682 		 * misrepresented, so we don't have to disable preemption here.
683 		 */
684 		last = raw_cpu_ptr(stats->last);
685 		stats_aux->merged =
686 			(bi_sector == (READ_ONCE(last->last_sector) &&
687 				       ((bi_rw == WRITE) ==
688 					(READ_ONCE(last->last_rw) == WRITE))
689 				       ));
690 		WRITE_ONCE(last->last_sector, end_sector);
691 		WRITE_ONCE(last->last_rw, bi_rw);
692 	} else
693 		duration_jiffies = jiffies - start_time;
694 
695 	rcu_read_lock();
696 
697 	got_precise_time = false;
698 	list_for_each_entry_rcu(s, &stats->list, list_entry) {
699 		if (s->stat_flags & STAT_PRECISE_TIMESTAMPS && !got_precise_time) {
700 			/* start (!end) duration_ns is set by DM core's alloc_io() */
701 			if (end)
702 				stats_aux->duration_ns = ktime_to_ns(ktime_get()) - stats_aux->duration_ns;
703 			got_precise_time = true;
704 		}
705 		__dm_stat_bio(s, bi_rw, bi_sector, end_sector, end, duration_jiffies, stats_aux);
706 	}
707 
708 	rcu_read_unlock();
709 }
710 
__dm_stat_init_temporary_percpu_totals(struct dm_stat_shared * shared,struct dm_stat * s,size_t x)711 static void __dm_stat_init_temporary_percpu_totals(struct dm_stat_shared *shared,
712 						   struct dm_stat *s, size_t x)
713 {
714 	int cpu;
715 	struct dm_stat_percpu *p;
716 
717 	local_irq_disable();
718 	p = &s->stat_percpu[smp_processor_id()][x];
719 	dm_stat_round(s, shared, p);
720 	local_irq_enable();
721 
722 	shared->tmp.sectors[READ] = 0;
723 	shared->tmp.sectors[WRITE] = 0;
724 	shared->tmp.ios[READ] = 0;
725 	shared->tmp.ios[WRITE] = 0;
726 	shared->tmp.merges[READ] = 0;
727 	shared->tmp.merges[WRITE] = 0;
728 	shared->tmp.ticks[READ] = 0;
729 	shared->tmp.ticks[WRITE] = 0;
730 	shared->tmp.io_ticks[READ] = 0;
731 	shared->tmp.io_ticks[WRITE] = 0;
732 	shared->tmp.io_ticks_total = 0;
733 	shared->tmp.time_in_queue = 0;
734 
735 	if (s->n_histogram_entries)
736 		memset(shared->tmp.histogram, 0, (s->n_histogram_entries + 1) * sizeof(unsigned long long));
737 
738 	for_each_possible_cpu(cpu) {
739 		p = &s->stat_percpu[cpu][x];
740 		shared->tmp.sectors[READ] += READ_ONCE(p->sectors[READ]);
741 		shared->tmp.sectors[WRITE] += READ_ONCE(p->sectors[WRITE]);
742 		shared->tmp.ios[READ] += READ_ONCE(p->ios[READ]);
743 		shared->tmp.ios[WRITE] += READ_ONCE(p->ios[WRITE]);
744 		shared->tmp.merges[READ] += READ_ONCE(p->merges[READ]);
745 		shared->tmp.merges[WRITE] += READ_ONCE(p->merges[WRITE]);
746 		shared->tmp.ticks[READ] += READ_ONCE(p->ticks[READ]);
747 		shared->tmp.ticks[WRITE] += READ_ONCE(p->ticks[WRITE]);
748 		shared->tmp.io_ticks[READ] += READ_ONCE(p->io_ticks[READ]);
749 		shared->tmp.io_ticks[WRITE] += READ_ONCE(p->io_ticks[WRITE]);
750 		shared->tmp.io_ticks_total += READ_ONCE(p->io_ticks_total);
751 		shared->tmp.time_in_queue += READ_ONCE(p->time_in_queue);
752 		if (s->n_histogram_entries) {
753 			unsigned int i;
754 
755 			for (i = 0; i < s->n_histogram_entries + 1; i++)
756 				shared->tmp.histogram[i] += READ_ONCE(p->histogram[i]);
757 		}
758 	}
759 }
760 
__dm_stat_clear(struct dm_stat * s,size_t idx_start,size_t idx_end,bool init_tmp_percpu_totals)761 static void __dm_stat_clear(struct dm_stat *s, size_t idx_start, size_t idx_end,
762 			    bool init_tmp_percpu_totals)
763 {
764 	size_t x;
765 	struct dm_stat_shared *shared;
766 	struct dm_stat_percpu *p;
767 
768 	for (x = idx_start; x < idx_end; x++) {
769 		shared = &s->stat_shared[x];
770 		if (init_tmp_percpu_totals)
771 			__dm_stat_init_temporary_percpu_totals(shared, s, x);
772 		local_irq_disable();
773 		p = &s->stat_percpu[smp_processor_id()][x];
774 		p->sectors[READ] -= shared->tmp.sectors[READ];
775 		p->sectors[WRITE] -= shared->tmp.sectors[WRITE];
776 		p->ios[READ] -= shared->tmp.ios[READ];
777 		p->ios[WRITE] -= shared->tmp.ios[WRITE];
778 		p->merges[READ] -= shared->tmp.merges[READ];
779 		p->merges[WRITE] -= shared->tmp.merges[WRITE];
780 		p->ticks[READ] -= shared->tmp.ticks[READ];
781 		p->ticks[WRITE] -= shared->tmp.ticks[WRITE];
782 		p->io_ticks[READ] -= shared->tmp.io_ticks[READ];
783 		p->io_ticks[WRITE] -= shared->tmp.io_ticks[WRITE];
784 		p->io_ticks_total -= shared->tmp.io_ticks_total;
785 		p->time_in_queue -= shared->tmp.time_in_queue;
786 		local_irq_enable();
787 		if (s->n_histogram_entries) {
788 			unsigned int i;
789 
790 			for (i = 0; i < s->n_histogram_entries + 1; i++) {
791 				local_irq_disable();
792 				p = &s->stat_percpu[smp_processor_id()][x];
793 				p->histogram[i] -= shared->tmp.histogram[i];
794 				local_irq_enable();
795 			}
796 		}
797 		cond_resched();
798 	}
799 }
800 
dm_stats_clear(struct dm_stats * stats,int id)801 static int dm_stats_clear(struct dm_stats *stats, int id)
802 {
803 	struct dm_stat *s;
804 
805 	mutex_lock(&stats->mutex);
806 
807 	s = __dm_stats_find(stats, id);
808 	if (!s) {
809 		mutex_unlock(&stats->mutex);
810 		return -ENOENT;
811 	}
812 
813 	__dm_stat_clear(s, 0, s->n_entries, true);
814 
815 	mutex_unlock(&stats->mutex);
816 
817 	return 1;
818 }
819 
820 /*
821  * This is like jiffies_to_msec, but works for 64-bit values.
822  */
dm_jiffies_to_msec64(struct dm_stat * s,unsigned long long j)823 static unsigned long long dm_jiffies_to_msec64(struct dm_stat *s, unsigned long long j)
824 {
825 	unsigned long long result;
826 	unsigned int mult;
827 
828 	if (s->stat_flags & STAT_PRECISE_TIMESTAMPS)
829 		return j;
830 
831 	result = 0;
832 	if (j)
833 		result = jiffies_to_msecs(j & 0x3fffff);
834 	if (j >= 1 << 22) {
835 		mult = jiffies_to_msecs(1 << 22);
836 		result += (unsigned long long)mult * (unsigned long long)jiffies_to_msecs((j >> 22) & 0x3fffff);
837 	}
838 	if (j >= 1ULL << 44)
839 		result += (unsigned long long)mult * (unsigned long long)mult * (unsigned long long)jiffies_to_msecs(j >> 44);
840 
841 	return result;
842 }
843 
dm_stats_print(struct dm_stats * stats,int id,size_t idx_start,size_t idx_len,bool clear,char * result,unsigned int maxlen)844 static int dm_stats_print(struct dm_stats *stats, int id,
845 			  size_t idx_start, size_t idx_len,
846 			  bool clear, char *result, unsigned int maxlen)
847 {
848 	unsigned int sz = 0;
849 	struct dm_stat *s;
850 	size_t x;
851 	sector_t start, end, step;
852 	size_t idx_end;
853 	struct dm_stat_shared *shared;
854 
855 	/*
856 	 * Output format:
857 	 *   <start_sector>+<length> counters
858 	 */
859 
860 	mutex_lock(&stats->mutex);
861 
862 	s = __dm_stats_find(stats, id);
863 	if (!s) {
864 		mutex_unlock(&stats->mutex);
865 		return -ENOENT;
866 	}
867 
868 	idx_end = idx_start + idx_len;
869 	if (idx_end < idx_start ||
870 	    idx_end > s->n_entries)
871 		idx_end = s->n_entries;
872 
873 	if (idx_start > idx_end)
874 		idx_start = idx_end;
875 
876 	step = s->step;
877 	start = s->start + (step * idx_start);
878 
879 	for (x = idx_start; x < idx_end; x++, start = end) {
880 		shared = &s->stat_shared[x];
881 		end = start + step;
882 		if (unlikely(end > s->end))
883 			end = s->end;
884 
885 		__dm_stat_init_temporary_percpu_totals(shared, s, x);
886 
887 		DMEMIT("%llu+%llu %llu %llu %llu %llu %llu %llu %llu %llu %d %llu %llu %llu %llu",
888 		       (unsigned long long)start,
889 		       (unsigned long long)step,
890 		       shared->tmp.ios[READ],
891 		       shared->tmp.merges[READ],
892 		       shared->tmp.sectors[READ],
893 		       dm_jiffies_to_msec64(s, shared->tmp.ticks[READ]),
894 		       shared->tmp.ios[WRITE],
895 		       shared->tmp.merges[WRITE],
896 		       shared->tmp.sectors[WRITE],
897 		       dm_jiffies_to_msec64(s, shared->tmp.ticks[WRITE]),
898 		       dm_stat_in_flight(shared),
899 		       dm_jiffies_to_msec64(s, shared->tmp.io_ticks_total),
900 		       dm_jiffies_to_msec64(s, shared->tmp.time_in_queue),
901 		       dm_jiffies_to_msec64(s, shared->tmp.io_ticks[READ]),
902 		       dm_jiffies_to_msec64(s, shared->tmp.io_ticks[WRITE]));
903 		if (s->n_histogram_entries) {
904 			unsigned int i;
905 
906 			for (i = 0; i < s->n_histogram_entries + 1; i++)
907 				DMEMIT("%s%llu", !i ? " " : ":", shared->tmp.histogram[i]);
908 		}
909 		DMEMIT("\n");
910 
911 		if (unlikely(sz + 1 >= maxlen))
912 			goto buffer_overflow;
913 
914 		cond_resched();
915 	}
916 
917 	if (clear)
918 		__dm_stat_clear(s, idx_start, idx_end, false);
919 
920 buffer_overflow:
921 	mutex_unlock(&stats->mutex);
922 
923 	return 1;
924 }
925 
dm_stats_set_aux(struct dm_stats * stats,int id,const char * aux_data)926 static int dm_stats_set_aux(struct dm_stats *stats, int id, const char *aux_data)
927 {
928 	struct dm_stat *s;
929 	const char *new_aux_data;
930 
931 	mutex_lock(&stats->mutex);
932 
933 	s = __dm_stats_find(stats, id);
934 	if (!s) {
935 		mutex_unlock(&stats->mutex);
936 		return -ENOENT;
937 	}
938 
939 	new_aux_data = kstrdup(aux_data, GFP_KERNEL);
940 	if (!new_aux_data) {
941 		mutex_unlock(&stats->mutex);
942 		return -ENOMEM;
943 	}
944 
945 	kfree(s->aux_data);
946 	s->aux_data = new_aux_data;
947 
948 	mutex_unlock(&stats->mutex);
949 
950 	return 0;
951 }
952 
parse_histogram(const char * h,unsigned int * n_histogram_entries,unsigned long long ** histogram_boundaries)953 static int parse_histogram(const char *h, unsigned int *n_histogram_entries,
954 			   unsigned long long **histogram_boundaries)
955 {
956 	const char *q;
957 	unsigned int n;
958 	unsigned long long last;
959 
960 	*n_histogram_entries = 1;
961 	for (q = h; *q; q++)
962 		if (*q == ',')
963 			(*n_histogram_entries)++;
964 
965 	*histogram_boundaries = kmalloc_array(*n_histogram_entries,
966 					      sizeof(unsigned long long),
967 					      GFP_KERNEL);
968 	if (!*histogram_boundaries)
969 		return -ENOMEM;
970 
971 	n = 0;
972 	last = 0;
973 	while (1) {
974 		unsigned long long hi;
975 		int s;
976 		char ch;
977 
978 		s = sscanf(h, "%llu%c", &hi, &ch);
979 		if (!s || (s == 2 && ch != ','))
980 			return -EINVAL;
981 		if (hi <= last)
982 			return -EINVAL;
983 		last = hi;
984 		(*histogram_boundaries)[n] = hi;
985 		if (s == 1)
986 			return 0;
987 		h = strchr(h, ',') + 1;
988 		n++;
989 	}
990 }
991 
message_stats_create(struct mapped_device * md,unsigned int argc,char ** argv,char * result,unsigned int maxlen)992 static int message_stats_create(struct mapped_device *md,
993 				unsigned int argc, char **argv,
994 				char *result, unsigned int maxlen)
995 {
996 	int r;
997 	int id;
998 	char dummy;
999 	unsigned long long start, end, len, step;
1000 	unsigned int divisor;
1001 	const char *program_id, *aux_data;
1002 	unsigned int stat_flags = 0;
1003 	unsigned int n_histogram_entries = 0;
1004 	unsigned long long *histogram_boundaries = NULL;
1005 	struct dm_arg_set as, as_backup;
1006 	const char *a;
1007 	unsigned int feature_args;
1008 
1009 	/*
1010 	 * Input format:
1011 	 *   <range> <step> [<extra_parameters> <parameters>] [<program_id> [<aux_data>]]
1012 	 */
1013 
1014 	if (argc < 3)
1015 		goto ret_einval;
1016 
1017 	as.argc = argc;
1018 	as.argv = argv;
1019 	dm_consume_args(&as, 1);
1020 
1021 	a = dm_shift_arg(&as);
1022 	if (!strcmp(a, "-")) {
1023 		start = 0;
1024 		len = dm_get_size(md);
1025 		if (!len)
1026 			len = 1;
1027 	} else if (sscanf(a, "%llu+%llu%c", &start, &len, &dummy) != 2 ||
1028 		   start != (sector_t)start || len != (sector_t)len)
1029 		goto ret_einval;
1030 
1031 	end = start + len;
1032 	if (start >= end)
1033 		goto ret_einval;
1034 
1035 	a = dm_shift_arg(&as);
1036 	if (sscanf(a, "/%u%c", &divisor, &dummy) == 1) {
1037 		if (!divisor)
1038 			return -EINVAL;
1039 		step = end - start;
1040 		if (do_div(step, divisor))
1041 			step++;
1042 		if (!step)
1043 			step = 1;
1044 	} else if (sscanf(a, "%llu%c", &step, &dummy) != 1 ||
1045 		   step != (sector_t)step || !step)
1046 		goto ret_einval;
1047 
1048 	as_backup = as;
1049 	a = dm_shift_arg(&as);
1050 	if (a && sscanf(a, "%u%c", &feature_args, &dummy) == 1) {
1051 		while (feature_args--) {
1052 			a = dm_shift_arg(&as);
1053 			if (!a)
1054 				goto ret_einval;
1055 			if (!strcasecmp(a, "precise_timestamps"))
1056 				stat_flags |= STAT_PRECISE_TIMESTAMPS;
1057 			else if (!strncasecmp(a, "histogram:", 10)) {
1058 				if (n_histogram_entries)
1059 					goto ret_einval;
1060 				r = parse_histogram(a + 10, &n_histogram_entries, &histogram_boundaries);
1061 				if (r)
1062 					goto ret;
1063 			} else
1064 				goto ret_einval;
1065 		}
1066 	} else {
1067 		as = as_backup;
1068 	}
1069 
1070 	program_id = "-";
1071 	aux_data = "-";
1072 
1073 	a = dm_shift_arg(&as);
1074 	if (a)
1075 		program_id = a;
1076 
1077 	a = dm_shift_arg(&as);
1078 	if (a)
1079 		aux_data = a;
1080 
1081 	if (as.argc)
1082 		goto ret_einval;
1083 
1084 	/*
1085 	 * If a buffer overflow happens after we created the region,
1086 	 * it's too late (the userspace would retry with a larger
1087 	 * buffer, but the region id that caused the overflow is already
1088 	 * leaked).  So we must detect buffer overflow in advance.
1089 	 */
1090 	snprintf(result, maxlen, "%d", INT_MAX);
1091 	if (dm_message_test_buffer_overflow(result, maxlen)) {
1092 		r = 1;
1093 		goto ret;
1094 	}
1095 
1096 	id = dm_stats_create(dm_get_stats(md), start, end, step, stat_flags,
1097 			     n_histogram_entries, histogram_boundaries, program_id, aux_data,
1098 			     dm_internal_suspend_fast, dm_internal_resume_fast, md);
1099 	if (id < 0) {
1100 		r = id;
1101 		goto ret;
1102 	}
1103 
1104 	snprintf(result, maxlen, "%d", id);
1105 
1106 	r = 1;
1107 	goto ret;
1108 
1109 ret_einval:
1110 	r = -EINVAL;
1111 ret:
1112 	kfree(histogram_boundaries);
1113 	return r;
1114 }
1115 
message_stats_delete(struct mapped_device * md,unsigned int argc,char ** argv)1116 static int message_stats_delete(struct mapped_device *md,
1117 				unsigned int argc, char **argv)
1118 {
1119 	int id;
1120 	char dummy;
1121 
1122 	if (argc != 2)
1123 		return -EINVAL;
1124 
1125 	if (sscanf(argv[1], "%d%c", &id, &dummy) != 1 || id < 0)
1126 		return -EINVAL;
1127 
1128 	return dm_stats_delete(dm_get_stats(md), id);
1129 }
1130 
message_stats_clear(struct mapped_device * md,unsigned int argc,char ** argv)1131 static int message_stats_clear(struct mapped_device *md,
1132 			       unsigned int argc, char **argv)
1133 {
1134 	int id;
1135 	char dummy;
1136 
1137 	if (argc != 2)
1138 		return -EINVAL;
1139 
1140 	if (sscanf(argv[1], "%d%c", &id, &dummy) != 1 || id < 0)
1141 		return -EINVAL;
1142 
1143 	return dm_stats_clear(dm_get_stats(md), id);
1144 }
1145 
message_stats_list(struct mapped_device * md,unsigned int argc,char ** argv,char * result,unsigned int maxlen)1146 static int message_stats_list(struct mapped_device *md,
1147 			      unsigned int argc, char **argv,
1148 			      char *result, unsigned int maxlen)
1149 {
1150 	int r;
1151 	const char *program = NULL;
1152 
1153 	if (argc < 1 || argc > 2)
1154 		return -EINVAL;
1155 
1156 	if (argc > 1) {
1157 		program = kstrdup(argv[1], GFP_KERNEL);
1158 		if (!program)
1159 			return -ENOMEM;
1160 	}
1161 
1162 	r = dm_stats_list(dm_get_stats(md), program, result, maxlen);
1163 
1164 	kfree(program);
1165 
1166 	return r;
1167 }
1168 
message_stats_print(struct mapped_device * md,unsigned int argc,char ** argv,bool clear,char * result,unsigned int maxlen)1169 static int message_stats_print(struct mapped_device *md,
1170 			       unsigned int argc, char **argv, bool clear,
1171 			       char *result, unsigned int maxlen)
1172 {
1173 	int id;
1174 	char dummy;
1175 	unsigned long idx_start = 0, idx_len = ULONG_MAX;
1176 
1177 	if (argc != 2 && argc != 4)
1178 		return -EINVAL;
1179 
1180 	if (sscanf(argv[1], "%d%c", &id, &dummy) != 1 || id < 0)
1181 		return -EINVAL;
1182 
1183 	if (argc > 3) {
1184 		if (strcmp(argv[2], "-") &&
1185 		    sscanf(argv[2], "%lu%c", &idx_start, &dummy) != 1)
1186 			return -EINVAL;
1187 		if (strcmp(argv[3], "-") &&
1188 		    sscanf(argv[3], "%lu%c", &idx_len, &dummy) != 1)
1189 			return -EINVAL;
1190 	}
1191 
1192 	return dm_stats_print(dm_get_stats(md), id, idx_start, idx_len, clear,
1193 			      result, maxlen);
1194 }
1195 
message_stats_set_aux(struct mapped_device * md,unsigned int argc,char ** argv)1196 static int message_stats_set_aux(struct mapped_device *md,
1197 				 unsigned int argc, char **argv)
1198 {
1199 	int id;
1200 	char dummy;
1201 
1202 	if (argc != 3)
1203 		return -EINVAL;
1204 
1205 	if (sscanf(argv[1], "%d%c", &id, &dummy) != 1 || id < 0)
1206 		return -EINVAL;
1207 
1208 	return dm_stats_set_aux(dm_get_stats(md), id, argv[2]);
1209 }
1210 
dm_stats_message(struct mapped_device * md,unsigned int argc,char ** argv,char * result,unsigned int maxlen)1211 int dm_stats_message(struct mapped_device *md, unsigned int argc, char **argv,
1212 		     char *result, unsigned int maxlen)
1213 {
1214 	int r;
1215 
1216 	/* All messages here must start with '@' */
1217 	if (!strcasecmp(argv[0], "@stats_create"))
1218 		r = message_stats_create(md, argc, argv, result, maxlen);
1219 	else if (!strcasecmp(argv[0], "@stats_delete"))
1220 		r = message_stats_delete(md, argc, argv);
1221 	else if (!strcasecmp(argv[0], "@stats_clear"))
1222 		r = message_stats_clear(md, argc, argv);
1223 	else if (!strcasecmp(argv[0], "@stats_list"))
1224 		r = message_stats_list(md, argc, argv, result, maxlen);
1225 	else if (!strcasecmp(argv[0], "@stats_print"))
1226 		r = message_stats_print(md, argc, argv, false, result, maxlen);
1227 	else if (!strcasecmp(argv[0], "@stats_print_clear"))
1228 		r = message_stats_print(md, argc, argv, true, result, maxlen);
1229 	else if (!strcasecmp(argv[0], "@stats_set_aux"))
1230 		r = message_stats_set_aux(md, argc, argv);
1231 	else
1232 		return 2; /* this wasn't a stats message */
1233 
1234 	if (r == -EINVAL)
1235 		DMCRIT("Invalid parameters for message %s", argv[0]);
1236 
1237 	return r;
1238 }
1239 
dm_statistics_init(void)1240 int __init dm_statistics_init(void)
1241 {
1242 	shared_memory_amount = 0;
1243 	dm_stat_need_rcu_barrier = 0;
1244 	return 0;
1245 }
1246 
dm_statistics_exit(void)1247 void dm_statistics_exit(void)
1248 {
1249 	if (dm_stat_need_rcu_barrier)
1250 		rcu_barrier();
1251 	if (WARN_ON(shared_memory_amount))
1252 		DMCRIT("shared_memory_amount leaked: %lu", shared_memory_amount);
1253 }
1254 
1255 module_param_named(stats_current_allocated_bytes, shared_memory_amount, ulong, 0444);
1256 MODULE_PARM_DESC(stats_current_allocated_bytes, "Memory currently used by statistics");
1257