xref: /openbmc/linux/kernel/sched/cputime.c (revision 31eeb6b0)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Simple CPU accounting cgroup controller
4  */
5 #include "sched.h"
6 
7 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
8 
9 /*
10  * There are no locks covering percpu hardirq/softirq time.
11  * They are only modified in vtime_account, on corresponding CPU
12  * with interrupts disabled. So, writes are safe.
13  * They are read and saved off onto struct rq in update_rq_clock().
14  * This may result in other CPU reading this CPU's irq time and can
15  * race with irq/vtime_account on this CPU. We would either get old
16  * or new value with a side effect of accounting a slice of irq time to wrong
17  * task when irq is in progress while we read rq->clock. That is a worthy
18  * compromise in place of having locks on each irq in account_system_time.
19  */
20 DEFINE_PER_CPU(struct irqtime, cpu_irqtime);
21 
22 static int sched_clock_irqtime;
23 
24 void enable_sched_clock_irqtime(void)
25 {
26 	sched_clock_irqtime = 1;
27 }
28 
29 void disable_sched_clock_irqtime(void)
30 {
31 	sched_clock_irqtime = 0;
32 }
33 
34 static void irqtime_account_delta(struct irqtime *irqtime, u64 delta,
35 				  enum cpu_usage_stat idx)
36 {
37 	u64 *cpustat = kcpustat_this_cpu->cpustat;
38 
39 	u64_stats_update_begin(&irqtime->sync);
40 	cpustat[idx] += delta;
41 	irqtime->total += delta;
42 	irqtime->tick_delta += delta;
43 	u64_stats_update_end(&irqtime->sync);
44 }
45 
46 /*
47  * Called after incrementing preempt_count on {soft,}irq_enter
48  * and before decrementing preempt_count on {soft,}irq_exit.
49  */
50 void irqtime_account_irq(struct task_struct *curr, unsigned int offset)
51 {
52 	struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
53 	unsigned int pc;
54 	s64 delta;
55 	int cpu;
56 
57 	if (!sched_clock_irqtime)
58 		return;
59 
60 	cpu = smp_processor_id();
61 	delta = sched_clock_cpu(cpu) - irqtime->irq_start_time;
62 	irqtime->irq_start_time += delta;
63 	pc = irq_count() - offset;
64 
65 	/*
66 	 * We do not account for softirq time from ksoftirqd here.
67 	 * We want to continue accounting softirq time to ksoftirqd thread
68 	 * in that case, so as not to confuse scheduler with a special task
69 	 * that do not consume any time, but still wants to run.
70 	 */
71 	if (pc & HARDIRQ_MASK)
72 		irqtime_account_delta(irqtime, delta, CPUTIME_IRQ);
73 	else if ((pc & SOFTIRQ_OFFSET) && curr != this_cpu_ksoftirqd())
74 		irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ);
75 }
76 
77 static u64 irqtime_tick_accounted(u64 maxtime)
78 {
79 	struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
80 	u64 delta;
81 
82 	delta = min(irqtime->tick_delta, maxtime);
83 	irqtime->tick_delta -= delta;
84 
85 	return delta;
86 }
87 
88 #else /* CONFIG_IRQ_TIME_ACCOUNTING */
89 
90 #define sched_clock_irqtime	(0)
91 
92 static u64 irqtime_tick_accounted(u64 dummy)
93 {
94 	return 0;
95 }
96 
97 #endif /* !CONFIG_IRQ_TIME_ACCOUNTING */
98 
99 static inline void task_group_account_field(struct task_struct *p, int index,
100 					    u64 tmp)
101 {
102 	/*
103 	 * Since all updates are sure to touch the root cgroup, we
104 	 * get ourselves ahead and touch it first. If the root cgroup
105 	 * is the only cgroup, then nothing else should be necessary.
106 	 *
107 	 */
108 	__this_cpu_add(kernel_cpustat.cpustat[index], tmp);
109 
110 	cgroup_account_cputime_field(p, index, tmp);
111 }
112 
113 /*
114  * Account user CPU time to a process.
115  * @p: the process that the CPU time gets accounted to
116  * @cputime: the CPU time spent in user space since the last update
117  */
118 void account_user_time(struct task_struct *p, u64 cputime)
119 {
120 	int index;
121 
122 	/* Add user time to process. */
123 	p->utime += cputime;
124 	account_group_user_time(p, cputime);
125 
126 	index = (task_nice(p) > 0) ? CPUTIME_NICE : CPUTIME_USER;
127 
128 	/* Add user time to cpustat. */
129 	task_group_account_field(p, index, cputime);
130 
131 	/* Account for user time used */
132 	acct_account_cputime(p);
133 }
134 
135 /*
136  * Account guest CPU time to a process.
137  * @p: the process that the CPU time gets accounted to
138  * @cputime: the CPU time spent in virtual machine since the last update
139  */
140 void account_guest_time(struct task_struct *p, u64 cputime)
141 {
142 	u64 *cpustat = kcpustat_this_cpu->cpustat;
143 
144 	/* Add guest time to process. */
145 	p->utime += cputime;
146 	account_group_user_time(p, cputime);
147 	p->gtime += cputime;
148 
149 	/* Add guest time to cpustat. */
150 	if (task_nice(p) > 0) {
151 		task_group_account_field(p, CPUTIME_NICE, cputime);
152 		cpustat[CPUTIME_GUEST_NICE] += cputime;
153 	} else {
154 		task_group_account_field(p, CPUTIME_USER, cputime);
155 		cpustat[CPUTIME_GUEST] += cputime;
156 	}
157 }
158 
159 /*
160  * Account system CPU time to a process and desired cpustat field
161  * @p: the process that the CPU time gets accounted to
162  * @cputime: the CPU time spent in kernel space since the last update
163  * @index: pointer to cpustat field that has to be updated
164  */
165 void account_system_index_time(struct task_struct *p,
166 			       u64 cputime, enum cpu_usage_stat index)
167 {
168 	/* Add system time to process. */
169 	p->stime += cputime;
170 	account_group_system_time(p, cputime);
171 
172 	/* Add system time to cpustat. */
173 	task_group_account_field(p, index, cputime);
174 
175 	/* Account for system time used */
176 	acct_account_cputime(p);
177 }
178 
179 /*
180  * Account system CPU time to a process.
181  * @p: the process that the CPU time gets accounted to
182  * @hardirq_offset: the offset to subtract from hardirq_count()
183  * @cputime: the CPU time spent in kernel space since the last update
184  */
185 void account_system_time(struct task_struct *p, int hardirq_offset, u64 cputime)
186 {
187 	int index;
188 
189 	if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
190 		account_guest_time(p, cputime);
191 		return;
192 	}
193 
194 	if (hardirq_count() - hardirq_offset)
195 		index = CPUTIME_IRQ;
196 	else if (in_serving_softirq())
197 		index = CPUTIME_SOFTIRQ;
198 	else
199 		index = CPUTIME_SYSTEM;
200 
201 	account_system_index_time(p, cputime, index);
202 }
203 
204 /*
205  * Account for involuntary wait time.
206  * @cputime: the CPU time spent in involuntary wait
207  */
208 void account_steal_time(u64 cputime)
209 {
210 	u64 *cpustat = kcpustat_this_cpu->cpustat;
211 
212 	cpustat[CPUTIME_STEAL] += cputime;
213 }
214 
215 /*
216  * Account for idle time.
217  * @cputime: the CPU time spent in idle wait
218  */
219 void account_idle_time(u64 cputime)
220 {
221 	u64 *cpustat = kcpustat_this_cpu->cpustat;
222 	struct rq *rq = this_rq();
223 
224 	if (atomic_read(&rq->nr_iowait) > 0)
225 		cpustat[CPUTIME_IOWAIT] += cputime;
226 	else
227 		cpustat[CPUTIME_IDLE] += cputime;
228 }
229 
230 /*
231  * When a guest is interrupted for a longer amount of time, missed clock
232  * ticks are not redelivered later. Due to that, this function may on
233  * occasion account more time than the calling functions think elapsed.
234  */
235 static __always_inline u64 steal_account_process_time(u64 maxtime)
236 {
237 #ifdef CONFIG_PARAVIRT
238 	if (static_key_false(&paravirt_steal_enabled)) {
239 		u64 steal;
240 
241 		steal = paravirt_steal_clock(smp_processor_id());
242 		steal -= this_rq()->prev_steal_time;
243 		steal = min(steal, maxtime);
244 		account_steal_time(steal);
245 		this_rq()->prev_steal_time += steal;
246 
247 		return steal;
248 	}
249 #endif
250 	return 0;
251 }
252 
253 /*
254  * Account how much elapsed time was spent in steal, irq, or softirq time.
255  */
256 static inline u64 account_other_time(u64 max)
257 {
258 	u64 accounted;
259 
260 	lockdep_assert_irqs_disabled();
261 
262 	accounted = steal_account_process_time(max);
263 
264 	if (accounted < max)
265 		accounted += irqtime_tick_accounted(max - accounted);
266 
267 	return accounted;
268 }
269 
270 #ifdef CONFIG_64BIT
271 static inline u64 read_sum_exec_runtime(struct task_struct *t)
272 {
273 	return t->se.sum_exec_runtime;
274 }
275 #else
276 static u64 read_sum_exec_runtime(struct task_struct *t)
277 {
278 	u64 ns;
279 	struct rq_flags rf;
280 	struct rq *rq;
281 
282 	rq = task_rq_lock(t, &rf);
283 	ns = t->se.sum_exec_runtime;
284 	task_rq_unlock(rq, t, &rf);
285 
286 	return ns;
287 }
288 #endif
289 
290 /*
291  * Accumulate raw cputime values of dead tasks (sig->[us]time) and live
292  * tasks (sum on group iteration) belonging to @tsk's group.
293  */
294 void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times)
295 {
296 	struct signal_struct *sig = tsk->signal;
297 	u64 utime, stime;
298 	struct task_struct *t;
299 	unsigned int seq, nextseq;
300 	unsigned long flags;
301 
302 	/*
303 	 * Update current task runtime to account pending time since last
304 	 * scheduler action or thread_group_cputime() call. This thread group
305 	 * might have other running tasks on different CPUs, but updating
306 	 * their runtime can affect syscall performance, so we skip account
307 	 * those pending times and rely only on values updated on tick or
308 	 * other scheduler action.
309 	 */
310 	if (same_thread_group(current, tsk))
311 		(void) task_sched_runtime(current);
312 
313 	rcu_read_lock();
314 	/* Attempt a lockless read on the first round. */
315 	nextseq = 0;
316 	do {
317 		seq = nextseq;
318 		flags = read_seqbegin_or_lock_irqsave(&sig->stats_lock, &seq);
319 		times->utime = sig->utime;
320 		times->stime = sig->stime;
321 		times->sum_exec_runtime = sig->sum_sched_runtime;
322 
323 		for_each_thread(tsk, t) {
324 			task_cputime(t, &utime, &stime);
325 			times->utime += utime;
326 			times->stime += stime;
327 			times->sum_exec_runtime += read_sum_exec_runtime(t);
328 		}
329 		/* If lockless access failed, take the lock. */
330 		nextseq = 1;
331 	} while (need_seqretry(&sig->stats_lock, seq));
332 	done_seqretry_irqrestore(&sig->stats_lock, seq, flags);
333 	rcu_read_unlock();
334 }
335 
336 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
337 /*
338  * Account a tick to a process and cpustat
339  * @p: the process that the CPU time gets accounted to
340  * @user_tick: is the tick from userspace
341  * @rq: the pointer to rq
342  *
343  * Tick demultiplexing follows the order
344  * - pending hardirq update
345  * - pending softirq update
346  * - user_time
347  * - idle_time
348  * - system time
349  *   - check for guest_time
350  *   - else account as system_time
351  *
352  * Check for hardirq is done both for system and user time as there is
353  * no timer going off while we are on hardirq and hence we may never get an
354  * opportunity to update it solely in system time.
355  * p->stime and friends are only updated on system time and not on irq
356  * softirq as those do not count in task exec_runtime any more.
357  */
358 static void irqtime_account_process_tick(struct task_struct *p, int user_tick,
359 					 int ticks)
360 {
361 	u64 other, cputime = TICK_NSEC * ticks;
362 
363 	/*
364 	 * When returning from idle, many ticks can get accounted at
365 	 * once, including some ticks of steal, irq, and softirq time.
366 	 * Subtract those ticks from the amount of time accounted to
367 	 * idle, or potentially user or system time. Due to rounding,
368 	 * other time can exceed ticks occasionally.
369 	 */
370 	other = account_other_time(ULONG_MAX);
371 	if (other >= cputime)
372 		return;
373 
374 	cputime -= other;
375 
376 	if (this_cpu_ksoftirqd() == p) {
377 		/*
378 		 * ksoftirqd time do not get accounted in cpu_softirq_time.
379 		 * So, we have to handle it separately here.
380 		 * Also, p->stime needs to be updated for ksoftirqd.
381 		 */
382 		account_system_index_time(p, cputime, CPUTIME_SOFTIRQ);
383 	} else if (user_tick) {
384 		account_user_time(p, cputime);
385 	} else if (p == this_rq()->idle) {
386 		account_idle_time(cputime);
387 	} else if (p->flags & PF_VCPU) { /* System time or guest time */
388 		account_guest_time(p, cputime);
389 	} else {
390 		account_system_index_time(p, cputime, CPUTIME_SYSTEM);
391 	}
392 }
393 
394 static void irqtime_account_idle_ticks(int ticks)
395 {
396 	irqtime_account_process_tick(current, 0, ticks);
397 }
398 #else /* CONFIG_IRQ_TIME_ACCOUNTING */
399 static inline void irqtime_account_idle_ticks(int ticks) { }
400 static inline void irqtime_account_process_tick(struct task_struct *p, int user_tick,
401 						int nr_ticks) { }
402 #endif /* CONFIG_IRQ_TIME_ACCOUNTING */
403 
404 /*
405  * Use precise platform statistics if available:
406  */
407 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
408 
409 # ifndef __ARCH_HAS_VTIME_TASK_SWITCH
410 void vtime_task_switch(struct task_struct *prev)
411 {
412 	if (is_idle_task(prev))
413 		vtime_account_idle(prev);
414 	else
415 		vtime_account_kernel(prev);
416 
417 	vtime_flush(prev);
418 	arch_vtime_task_switch(prev);
419 }
420 # endif
421 
422 void vtime_account_irq(struct task_struct *tsk, unsigned int offset)
423 {
424 	unsigned int pc = irq_count() - offset;
425 
426 	if (pc & HARDIRQ_OFFSET) {
427 		vtime_account_hardirq(tsk);
428 	} else if (pc & SOFTIRQ_OFFSET) {
429 		vtime_account_softirq(tsk);
430 	} else if (!IS_ENABLED(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) &&
431 		   is_idle_task(tsk)) {
432 		vtime_account_idle(tsk);
433 	} else {
434 		vtime_account_kernel(tsk);
435 	}
436 }
437 
438 void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,
439 		    u64 *ut, u64 *st)
440 {
441 	*ut = curr->utime;
442 	*st = curr->stime;
443 }
444 
445 void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
446 {
447 	*ut = p->utime;
448 	*st = p->stime;
449 }
450 EXPORT_SYMBOL_GPL(task_cputime_adjusted);
451 
452 void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
453 {
454 	struct task_cputime cputime;
455 
456 	thread_group_cputime(p, &cputime);
457 
458 	*ut = cputime.utime;
459 	*st = cputime.stime;
460 }
461 
462 #else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE: */
463 
464 /*
465  * Account a single tick of CPU time.
466  * @p: the process that the CPU time gets accounted to
467  * @user_tick: indicates if the tick is a user or a system tick
468  */
469 void account_process_tick(struct task_struct *p, int user_tick)
470 {
471 	u64 cputime, steal;
472 
473 	if (vtime_accounting_enabled_this_cpu())
474 		return;
475 
476 	if (sched_clock_irqtime) {
477 		irqtime_account_process_tick(p, user_tick, 1);
478 		return;
479 	}
480 
481 	cputime = TICK_NSEC;
482 	steal = steal_account_process_time(ULONG_MAX);
483 
484 	if (steal >= cputime)
485 		return;
486 
487 	cputime -= steal;
488 
489 	if (user_tick)
490 		account_user_time(p, cputime);
491 	else if ((p != this_rq()->idle) || (irq_count() != HARDIRQ_OFFSET))
492 		account_system_time(p, HARDIRQ_OFFSET, cputime);
493 	else
494 		account_idle_time(cputime);
495 }
496 
497 /*
498  * Account multiple ticks of idle time.
499  * @ticks: number of stolen ticks
500  */
501 void account_idle_ticks(unsigned long ticks)
502 {
503 	u64 cputime, steal;
504 
505 	if (sched_clock_irqtime) {
506 		irqtime_account_idle_ticks(ticks);
507 		return;
508 	}
509 
510 	cputime = ticks * TICK_NSEC;
511 	steal = steal_account_process_time(ULONG_MAX);
512 
513 	if (steal >= cputime)
514 		return;
515 
516 	cputime -= steal;
517 	account_idle_time(cputime);
518 }
519 
520 /*
521  * Adjust tick based cputime random precision against scheduler runtime
522  * accounting.
523  *
524  * Tick based cputime accounting depend on random scheduling timeslices of a
525  * task to be interrupted or not by the timer.  Depending on these
526  * circumstances, the number of these interrupts may be over or
527  * under-optimistic, matching the real user and system cputime with a variable
528  * precision.
529  *
530  * Fix this by scaling these tick based values against the total runtime
531  * accounted by the CFS scheduler.
532  *
533  * This code provides the following guarantees:
534  *
535  *   stime + utime == rtime
536  *   stime_i+1 >= stime_i, utime_i+1 >= utime_i
537  *
538  * Assuming that rtime_i+1 >= rtime_i.
539  */
540 void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,
541 		    u64 *ut, u64 *st)
542 {
543 	u64 rtime, stime, utime;
544 	unsigned long flags;
545 
546 	/* Serialize concurrent callers such that we can honour our guarantees */
547 	raw_spin_lock_irqsave(&prev->lock, flags);
548 	rtime = curr->sum_exec_runtime;
549 
550 	/*
551 	 * This is possible under two circumstances:
552 	 *  - rtime isn't monotonic after all (a bug);
553 	 *  - we got reordered by the lock.
554 	 *
555 	 * In both cases this acts as a filter such that the rest of the code
556 	 * can assume it is monotonic regardless of anything else.
557 	 */
558 	if (prev->stime + prev->utime >= rtime)
559 		goto out;
560 
561 	stime = curr->stime;
562 	utime = curr->utime;
563 
564 	/*
565 	 * If either stime or utime are 0, assume all runtime is userspace.
566 	 * Once a task gets some ticks, the monotonicity code at 'update:'
567 	 * will ensure things converge to the observed ratio.
568 	 */
569 	if (stime == 0) {
570 		utime = rtime;
571 		goto update;
572 	}
573 
574 	if (utime == 0) {
575 		stime = rtime;
576 		goto update;
577 	}
578 
579 	stime = mul_u64_u64_div_u64(stime, rtime, stime + utime);
580 
581 update:
582 	/*
583 	 * Make sure stime doesn't go backwards; this preserves monotonicity
584 	 * for utime because rtime is monotonic.
585 	 *
586 	 *  utime_i+1 = rtime_i+1 - stime_i
587 	 *            = rtime_i+1 - (rtime_i - utime_i)
588 	 *            = (rtime_i+1 - rtime_i) + utime_i
589 	 *            >= utime_i
590 	 */
591 	if (stime < prev->stime)
592 		stime = prev->stime;
593 	utime = rtime - stime;
594 
595 	/*
596 	 * Make sure utime doesn't go backwards; this still preserves
597 	 * monotonicity for stime, analogous argument to above.
598 	 */
599 	if (utime < prev->utime) {
600 		utime = prev->utime;
601 		stime = rtime - utime;
602 	}
603 
604 	prev->stime = stime;
605 	prev->utime = utime;
606 out:
607 	*ut = prev->utime;
608 	*st = prev->stime;
609 	raw_spin_unlock_irqrestore(&prev->lock, flags);
610 }
611 
612 void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
613 {
614 	struct task_cputime cputime = {
615 		.sum_exec_runtime = p->se.sum_exec_runtime,
616 	};
617 
618 	if (task_cputime(p, &cputime.utime, &cputime.stime))
619 		cputime.sum_exec_runtime = task_sched_runtime(p);
620 	cputime_adjust(&cputime, &p->prev_cputime, ut, st);
621 }
622 EXPORT_SYMBOL_GPL(task_cputime_adjusted);
623 
624 void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
625 {
626 	struct task_cputime cputime;
627 
628 	thread_group_cputime(p, &cputime);
629 	cputime_adjust(&cputime, &p->signal->prev_cputime, ut, st);
630 }
631 #endif /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
632 
633 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
634 static u64 vtime_delta(struct vtime *vtime)
635 {
636 	unsigned long long clock;
637 
638 	clock = sched_clock();
639 	if (clock < vtime->starttime)
640 		return 0;
641 
642 	return clock - vtime->starttime;
643 }
644 
645 static u64 get_vtime_delta(struct vtime *vtime)
646 {
647 	u64 delta = vtime_delta(vtime);
648 	u64 other;
649 
650 	/*
651 	 * Unlike tick based timing, vtime based timing never has lost
652 	 * ticks, and no need for steal time accounting to make up for
653 	 * lost ticks. Vtime accounts a rounded version of actual
654 	 * elapsed time. Limit account_other_time to prevent rounding
655 	 * errors from causing elapsed vtime to go negative.
656 	 */
657 	other = account_other_time(delta);
658 	WARN_ON_ONCE(vtime->state == VTIME_INACTIVE);
659 	vtime->starttime += delta;
660 
661 	return delta - other;
662 }
663 
664 static void vtime_account_system(struct task_struct *tsk,
665 				 struct vtime *vtime)
666 {
667 	vtime->stime += get_vtime_delta(vtime);
668 	if (vtime->stime >= TICK_NSEC) {
669 		account_system_time(tsk, irq_count(), vtime->stime);
670 		vtime->stime = 0;
671 	}
672 }
673 
674 static void vtime_account_guest(struct task_struct *tsk,
675 				struct vtime *vtime)
676 {
677 	vtime->gtime += get_vtime_delta(vtime);
678 	if (vtime->gtime >= TICK_NSEC) {
679 		account_guest_time(tsk, vtime->gtime);
680 		vtime->gtime = 0;
681 	}
682 }
683 
684 static void __vtime_account_kernel(struct task_struct *tsk,
685 				   struct vtime *vtime)
686 {
687 	/* We might have scheduled out from guest path */
688 	if (vtime->state == VTIME_GUEST)
689 		vtime_account_guest(tsk, vtime);
690 	else
691 		vtime_account_system(tsk, vtime);
692 }
693 
694 void vtime_account_kernel(struct task_struct *tsk)
695 {
696 	struct vtime *vtime = &tsk->vtime;
697 
698 	if (!vtime_delta(vtime))
699 		return;
700 
701 	write_seqcount_begin(&vtime->seqcount);
702 	__vtime_account_kernel(tsk, vtime);
703 	write_seqcount_end(&vtime->seqcount);
704 }
705 
706 void vtime_user_enter(struct task_struct *tsk)
707 {
708 	struct vtime *vtime = &tsk->vtime;
709 
710 	write_seqcount_begin(&vtime->seqcount);
711 	vtime_account_system(tsk, vtime);
712 	vtime->state = VTIME_USER;
713 	write_seqcount_end(&vtime->seqcount);
714 }
715 
716 void vtime_user_exit(struct task_struct *tsk)
717 {
718 	struct vtime *vtime = &tsk->vtime;
719 
720 	write_seqcount_begin(&vtime->seqcount);
721 	vtime->utime += get_vtime_delta(vtime);
722 	if (vtime->utime >= TICK_NSEC) {
723 		account_user_time(tsk, vtime->utime);
724 		vtime->utime = 0;
725 	}
726 	vtime->state = VTIME_SYS;
727 	write_seqcount_end(&vtime->seqcount);
728 }
729 
730 void vtime_guest_enter(struct task_struct *tsk)
731 {
732 	struct vtime *vtime = &tsk->vtime;
733 	/*
734 	 * The flags must be updated under the lock with
735 	 * the vtime_starttime flush and update.
736 	 * That enforces a right ordering and update sequence
737 	 * synchronization against the reader (task_gtime())
738 	 * that can thus safely catch up with a tickless delta.
739 	 */
740 	write_seqcount_begin(&vtime->seqcount);
741 	vtime_account_system(tsk, vtime);
742 	tsk->flags |= PF_VCPU;
743 	vtime->state = VTIME_GUEST;
744 	write_seqcount_end(&vtime->seqcount);
745 }
746 EXPORT_SYMBOL_GPL(vtime_guest_enter);
747 
748 void vtime_guest_exit(struct task_struct *tsk)
749 {
750 	struct vtime *vtime = &tsk->vtime;
751 
752 	write_seqcount_begin(&vtime->seqcount);
753 	vtime_account_guest(tsk, vtime);
754 	tsk->flags &= ~PF_VCPU;
755 	vtime->state = VTIME_SYS;
756 	write_seqcount_end(&vtime->seqcount);
757 }
758 EXPORT_SYMBOL_GPL(vtime_guest_exit);
759 
760 void vtime_account_idle(struct task_struct *tsk)
761 {
762 	account_idle_time(get_vtime_delta(&tsk->vtime));
763 }
764 
765 void vtime_task_switch_generic(struct task_struct *prev)
766 {
767 	struct vtime *vtime = &prev->vtime;
768 
769 	write_seqcount_begin(&vtime->seqcount);
770 	if (vtime->state == VTIME_IDLE)
771 		vtime_account_idle(prev);
772 	else
773 		__vtime_account_kernel(prev, vtime);
774 	vtime->state = VTIME_INACTIVE;
775 	vtime->cpu = -1;
776 	write_seqcount_end(&vtime->seqcount);
777 
778 	vtime = &current->vtime;
779 
780 	write_seqcount_begin(&vtime->seqcount);
781 	if (is_idle_task(current))
782 		vtime->state = VTIME_IDLE;
783 	else if (current->flags & PF_VCPU)
784 		vtime->state = VTIME_GUEST;
785 	else
786 		vtime->state = VTIME_SYS;
787 	vtime->starttime = sched_clock();
788 	vtime->cpu = smp_processor_id();
789 	write_seqcount_end(&vtime->seqcount);
790 }
791 
792 void vtime_init_idle(struct task_struct *t, int cpu)
793 {
794 	struct vtime *vtime = &t->vtime;
795 	unsigned long flags;
796 
797 	local_irq_save(flags);
798 	write_seqcount_begin(&vtime->seqcount);
799 	vtime->state = VTIME_IDLE;
800 	vtime->starttime = sched_clock();
801 	vtime->cpu = cpu;
802 	write_seqcount_end(&vtime->seqcount);
803 	local_irq_restore(flags);
804 }
805 
806 u64 task_gtime(struct task_struct *t)
807 {
808 	struct vtime *vtime = &t->vtime;
809 	unsigned int seq;
810 	u64 gtime;
811 
812 	if (!vtime_accounting_enabled())
813 		return t->gtime;
814 
815 	do {
816 		seq = read_seqcount_begin(&vtime->seqcount);
817 
818 		gtime = t->gtime;
819 		if (vtime->state == VTIME_GUEST)
820 			gtime += vtime->gtime + vtime_delta(vtime);
821 
822 	} while (read_seqcount_retry(&vtime->seqcount, seq));
823 
824 	return gtime;
825 }
826 
827 /*
828  * Fetch cputime raw values from fields of task_struct and
829  * add up the pending nohz execution time since the last
830  * cputime snapshot.
831  */
832 bool task_cputime(struct task_struct *t, u64 *utime, u64 *stime)
833 {
834 	struct vtime *vtime = &t->vtime;
835 	unsigned int seq;
836 	u64 delta;
837 	int ret;
838 
839 	if (!vtime_accounting_enabled()) {
840 		*utime = t->utime;
841 		*stime = t->stime;
842 		return false;
843 	}
844 
845 	do {
846 		ret = false;
847 		seq = read_seqcount_begin(&vtime->seqcount);
848 
849 		*utime = t->utime;
850 		*stime = t->stime;
851 
852 		/* Task is sleeping or idle, nothing to add */
853 		if (vtime->state < VTIME_SYS)
854 			continue;
855 
856 		ret = true;
857 		delta = vtime_delta(vtime);
858 
859 		/*
860 		 * Task runs either in user (including guest) or kernel space,
861 		 * add pending nohz time to the right place.
862 		 */
863 		if (vtime->state == VTIME_SYS)
864 			*stime += vtime->stime + delta;
865 		else
866 			*utime += vtime->utime + delta;
867 	} while (read_seqcount_retry(&vtime->seqcount, seq));
868 
869 	return ret;
870 }
871 
872 static int vtime_state_fetch(struct vtime *vtime, int cpu)
873 {
874 	int state = READ_ONCE(vtime->state);
875 
876 	/*
877 	 * We raced against a context switch, fetch the
878 	 * kcpustat task again.
879 	 */
880 	if (vtime->cpu != cpu && vtime->cpu != -1)
881 		return -EAGAIN;
882 
883 	/*
884 	 * Two possible things here:
885 	 * 1) We are seeing the scheduling out task (prev) or any past one.
886 	 * 2) We are seeing the scheduling in task (next) but it hasn't
887 	 *    passed though vtime_task_switch() yet so the pending
888 	 *    cputime of the prev task may not be flushed yet.
889 	 *
890 	 * Case 1) is ok but 2) is not. So wait for a safe VTIME state.
891 	 */
892 	if (state == VTIME_INACTIVE)
893 		return -EAGAIN;
894 
895 	return state;
896 }
897 
898 static u64 kcpustat_user_vtime(struct vtime *vtime)
899 {
900 	if (vtime->state == VTIME_USER)
901 		return vtime->utime + vtime_delta(vtime);
902 	else if (vtime->state == VTIME_GUEST)
903 		return vtime->gtime + vtime_delta(vtime);
904 	return 0;
905 }
906 
907 static int kcpustat_field_vtime(u64 *cpustat,
908 				struct task_struct *tsk,
909 				enum cpu_usage_stat usage,
910 				int cpu, u64 *val)
911 {
912 	struct vtime *vtime = &tsk->vtime;
913 	unsigned int seq;
914 
915 	do {
916 		int state;
917 
918 		seq = read_seqcount_begin(&vtime->seqcount);
919 
920 		state = vtime_state_fetch(vtime, cpu);
921 		if (state < 0)
922 			return state;
923 
924 		*val = cpustat[usage];
925 
926 		/*
927 		 * Nice VS unnice cputime accounting may be inaccurate if
928 		 * the nice value has changed since the last vtime update.
929 		 * But proper fix would involve interrupting target on nice
930 		 * updates which is a no go on nohz_full (although the scheduler
931 		 * may still interrupt the target if rescheduling is needed...)
932 		 */
933 		switch (usage) {
934 		case CPUTIME_SYSTEM:
935 			if (state == VTIME_SYS)
936 				*val += vtime->stime + vtime_delta(vtime);
937 			break;
938 		case CPUTIME_USER:
939 			if (task_nice(tsk) <= 0)
940 				*val += kcpustat_user_vtime(vtime);
941 			break;
942 		case CPUTIME_NICE:
943 			if (task_nice(tsk) > 0)
944 				*val += kcpustat_user_vtime(vtime);
945 			break;
946 		case CPUTIME_GUEST:
947 			if (state == VTIME_GUEST && task_nice(tsk) <= 0)
948 				*val += vtime->gtime + vtime_delta(vtime);
949 			break;
950 		case CPUTIME_GUEST_NICE:
951 			if (state == VTIME_GUEST && task_nice(tsk) > 0)
952 				*val += vtime->gtime + vtime_delta(vtime);
953 			break;
954 		default:
955 			break;
956 		}
957 	} while (read_seqcount_retry(&vtime->seqcount, seq));
958 
959 	return 0;
960 }
961 
962 u64 kcpustat_field(struct kernel_cpustat *kcpustat,
963 		   enum cpu_usage_stat usage, int cpu)
964 {
965 	u64 *cpustat = kcpustat->cpustat;
966 	u64 val = cpustat[usage];
967 	struct rq *rq;
968 	int err;
969 
970 	if (!vtime_accounting_enabled_cpu(cpu))
971 		return val;
972 
973 	rq = cpu_rq(cpu);
974 
975 	for (;;) {
976 		struct task_struct *curr;
977 
978 		rcu_read_lock();
979 		curr = rcu_dereference(rq->curr);
980 		if (WARN_ON_ONCE(!curr)) {
981 			rcu_read_unlock();
982 			return cpustat[usage];
983 		}
984 
985 		err = kcpustat_field_vtime(cpustat, curr, usage, cpu, &val);
986 		rcu_read_unlock();
987 
988 		if (!err)
989 			return val;
990 
991 		cpu_relax();
992 	}
993 }
994 EXPORT_SYMBOL_GPL(kcpustat_field);
995 
996 static int kcpustat_cpu_fetch_vtime(struct kernel_cpustat *dst,
997 				    const struct kernel_cpustat *src,
998 				    struct task_struct *tsk, int cpu)
999 {
1000 	struct vtime *vtime = &tsk->vtime;
1001 	unsigned int seq;
1002 
1003 	do {
1004 		u64 *cpustat;
1005 		u64 delta;
1006 		int state;
1007 
1008 		seq = read_seqcount_begin(&vtime->seqcount);
1009 
1010 		state = vtime_state_fetch(vtime, cpu);
1011 		if (state < 0)
1012 			return state;
1013 
1014 		*dst = *src;
1015 		cpustat = dst->cpustat;
1016 
1017 		/* Task is sleeping, dead or idle, nothing to add */
1018 		if (state < VTIME_SYS)
1019 			continue;
1020 
1021 		delta = vtime_delta(vtime);
1022 
1023 		/*
1024 		 * Task runs either in user (including guest) or kernel space,
1025 		 * add pending nohz time to the right place.
1026 		 */
1027 		if (state == VTIME_SYS) {
1028 			cpustat[CPUTIME_SYSTEM] += vtime->stime + delta;
1029 		} else if (state == VTIME_USER) {
1030 			if (task_nice(tsk) > 0)
1031 				cpustat[CPUTIME_NICE] += vtime->utime + delta;
1032 			else
1033 				cpustat[CPUTIME_USER] += vtime->utime + delta;
1034 		} else {
1035 			WARN_ON_ONCE(state != VTIME_GUEST);
1036 			if (task_nice(tsk) > 0) {
1037 				cpustat[CPUTIME_GUEST_NICE] += vtime->gtime + delta;
1038 				cpustat[CPUTIME_NICE] += vtime->gtime + delta;
1039 			} else {
1040 				cpustat[CPUTIME_GUEST] += vtime->gtime + delta;
1041 				cpustat[CPUTIME_USER] += vtime->gtime + delta;
1042 			}
1043 		}
1044 	} while (read_seqcount_retry(&vtime->seqcount, seq));
1045 
1046 	return 0;
1047 }
1048 
1049 void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu)
1050 {
1051 	const struct kernel_cpustat *src = &kcpustat_cpu(cpu);
1052 	struct rq *rq;
1053 	int err;
1054 
1055 	if (!vtime_accounting_enabled_cpu(cpu)) {
1056 		*dst = *src;
1057 		return;
1058 	}
1059 
1060 	rq = cpu_rq(cpu);
1061 
1062 	for (;;) {
1063 		struct task_struct *curr;
1064 
1065 		rcu_read_lock();
1066 		curr = rcu_dereference(rq->curr);
1067 		if (WARN_ON_ONCE(!curr)) {
1068 			rcu_read_unlock();
1069 			*dst = *src;
1070 			return;
1071 		}
1072 
1073 		err = kcpustat_cpu_fetch_vtime(dst, src, curr, cpu);
1074 		rcu_read_unlock();
1075 
1076 		if (!err)
1077 			return;
1078 
1079 		cpu_relax();
1080 	}
1081 }
1082 EXPORT_SYMBOL_GPL(kcpustat_cpu_fetch);
1083 
1084 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */
1085