17170066eSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
2ca74e92bSShailabh Nagar /* delayacct.c - per-task delay accounting
3ca74e92bSShailabh Nagar *
4ca74e92bSShailabh Nagar * Copyright (C) Shailabh Nagar, IBM Corp. 2006
5ca74e92bSShailabh Nagar */
6ca74e92bSShailabh Nagar
7ca74e92bSShailabh Nagar #include <linux/sched.h>
89164bb4aSIngo Molnar #include <linux/sched/task.h>
932ef5517SIngo Molnar #include <linux/sched/cputime.h>
104b7a08a0SPeter Zijlstra #include <linux/sched/clock.h>
11ca74e92bSShailabh Nagar #include <linux/slab.h>
126952b61dSAlexey Dobriyan #include <linux/taskstats.h>
13ca74e92bSShailabh Nagar #include <linux/sysctl.h>
14ca74e92bSShailabh Nagar #include <linux/delayacct.h>
15c9aaa895SGlauber Costa #include <linux/module.h>
16ca74e92bSShailabh Nagar
17e4042ad4SPeter Zijlstra DEFINE_STATIC_KEY_FALSE(delayacct_key);
18e4042ad4SPeter Zijlstra int delayacct_on __read_mostly; /* Delay accounting turned on/off */
19e18b890bSChristoph Lameter struct kmem_cache *delayacct_cache;
20ca74e92bSShailabh Nagar
set_delayacct(bool enabled)210cd7c741SPeter Zijlstra static void set_delayacct(bool enabled)
220cd7c741SPeter Zijlstra {
230cd7c741SPeter Zijlstra if (enabled) {
240cd7c741SPeter Zijlstra static_branch_enable(&delayacct_key);
250cd7c741SPeter Zijlstra delayacct_on = 1;
260cd7c741SPeter Zijlstra } else {
270cd7c741SPeter Zijlstra delayacct_on = 0;
280cd7c741SPeter Zijlstra static_branch_disable(&delayacct_key);
290cd7c741SPeter Zijlstra }
300cd7c741SPeter Zijlstra }
310cd7c741SPeter Zijlstra
delayacct_setup_enable(char * str)32e4042ad4SPeter Zijlstra static int __init delayacct_setup_enable(char *str)
33ca74e92bSShailabh Nagar {
34e4042ad4SPeter Zijlstra delayacct_on = 1;
35ca74e92bSShailabh Nagar return 1;
36ca74e92bSShailabh Nagar }
37e4042ad4SPeter Zijlstra __setup("delayacct", delayacct_setup_enable);
38ca74e92bSShailabh Nagar
delayacct_init(void)39ca74e92bSShailabh Nagar void delayacct_init(void)
40ca74e92bSShailabh Nagar {
415d097056SVladimir Davydov delayacct_cache = KMEM_CACHE(task_delay_info, SLAB_PANIC|SLAB_ACCOUNT);
42ca74e92bSShailabh Nagar delayacct_tsk_init(&init_task);
430cd7c741SPeter Zijlstra set_delayacct(delayacct_on);
44ca74e92bSShailabh Nagar }
45ca74e92bSShailabh Nagar
460cd7c741SPeter Zijlstra #ifdef CONFIG_PROC_SYSCTL
sysctl_delayacct(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)471186618aStangmeng static int sysctl_delayacct(struct ctl_table *table, int write, void *buffer,
480cd7c741SPeter Zijlstra size_t *lenp, loff_t *ppos)
490cd7c741SPeter Zijlstra {
500cd7c741SPeter Zijlstra int state = delayacct_on;
510cd7c741SPeter Zijlstra struct ctl_table t;
520cd7c741SPeter Zijlstra int err;
530cd7c741SPeter Zijlstra
540cd7c741SPeter Zijlstra if (write && !capable(CAP_SYS_ADMIN))
550cd7c741SPeter Zijlstra return -EPERM;
560cd7c741SPeter Zijlstra
570cd7c741SPeter Zijlstra t = *table;
580cd7c741SPeter Zijlstra t.data = &state;
590cd7c741SPeter Zijlstra err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
600cd7c741SPeter Zijlstra if (err < 0)
610cd7c741SPeter Zijlstra return err;
620cd7c741SPeter Zijlstra if (write)
630cd7c741SPeter Zijlstra set_delayacct(state);
640cd7c741SPeter Zijlstra return err;
650cd7c741SPeter Zijlstra }
661186618aStangmeng
671186618aStangmeng static struct ctl_table kern_delayacct_table[] = {
681186618aStangmeng {
691186618aStangmeng .procname = "task_delayacct",
701186618aStangmeng .data = NULL,
711186618aStangmeng .maxlen = sizeof(unsigned int),
721186618aStangmeng .mode = 0644,
731186618aStangmeng .proc_handler = sysctl_delayacct,
741186618aStangmeng .extra1 = SYSCTL_ZERO,
751186618aStangmeng .extra2 = SYSCTL_ONE,
761186618aStangmeng },
771186618aStangmeng { }
781186618aStangmeng };
791186618aStangmeng
kernel_delayacct_sysctls_init(void)801186618aStangmeng static __init int kernel_delayacct_sysctls_init(void)
811186618aStangmeng {
821186618aStangmeng register_sysctl_init("kernel", kern_delayacct_table);
831186618aStangmeng return 0;
841186618aStangmeng }
851186618aStangmeng late_initcall(kernel_delayacct_sysctls_init);
860cd7c741SPeter Zijlstra #endif
870cd7c741SPeter Zijlstra
__delayacct_tsk_init(struct task_struct * tsk)88ca74e92bSShailabh Nagar void __delayacct_tsk_init(struct task_struct *tsk)
89ca74e92bSShailabh Nagar {
90e94b1766SChristoph Lameter tsk->delays = kmem_cache_zalloc(delayacct_cache, GFP_KERNEL);
91ca74e92bSShailabh Nagar if (tsk->delays)
9202acc80dSSebastian Andrzej Siewior raw_spin_lock_init(&tsk->delays->lock);
93ca74e92bSShailabh Nagar }
94ca74e92bSShailabh Nagar
95ca74e92bSShailabh Nagar /*
969667a23dSThomas Gleixner * Finish delay accounting for a statistic using its timestamps (@start),
979667a23dSThomas Gleixner * accumalator (@total) and @count
98ca74e92bSShailabh Nagar */
delayacct_end(raw_spinlock_t * lock,u64 * start,u64 * total,u32 * count)994b7a08a0SPeter Zijlstra static void delayacct_end(raw_spinlock_t *lock, u64 *start, u64 *total, u32 *count)
100ca74e92bSShailabh Nagar {
1014b7a08a0SPeter Zijlstra s64 ns = local_clock() - *start;
10264efade1SPeter Zijlstra unsigned long flags;
103ca74e92bSShailabh Nagar
1049667a23dSThomas Gleixner if (ns > 0) {
10502acc80dSSebastian Andrzej Siewior raw_spin_lock_irqsave(lock, flags);
106ca74e92bSShailabh Nagar *total += ns;
107ca74e92bSShailabh Nagar (*count)++;
10802acc80dSSebastian Andrzej Siewior raw_spin_unlock_irqrestore(lock, flags);
109ca74e92bSShailabh Nagar }
1109667a23dSThomas Gleixner }
111ca74e92bSShailabh Nagar
__delayacct_blkio_start(void)1120ff92245SShailabh Nagar void __delayacct_blkio_start(void)
1130ff92245SShailabh Nagar {
1144b7a08a0SPeter Zijlstra current->delays->blkio_start = local_clock();
1150ff92245SShailabh Nagar }
1160ff92245SShailabh Nagar
117c96f5471SJosh Snyder /*
118c96f5471SJosh Snyder * We cannot rely on the `current` macro, as we haven't yet switched back to
119c96f5471SJosh Snyder * the process being woken.
120c96f5471SJosh Snyder */
__delayacct_blkio_end(struct task_struct * p)121c96f5471SJosh Snyder void __delayacct_blkio_end(struct task_struct *p)
1220ff92245SShailabh Nagar {
123a3d5dc90SYang Yang delayacct_end(&p->delays->lock,
124a3d5dc90SYang Yang &p->delays->blkio_start,
125a3d5dc90SYang Yang &p->delays->blkio_delay,
126a3d5dc90SYang Yang &p->delays->blkio_count);
1270ff92245SShailabh Nagar }
1286f44993fSShailabh Nagar
delayacct_add_tsk(struct taskstats * d,struct task_struct * tsk)129e4042ad4SPeter Zijlstra int delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk)
1306f44993fSShailabh Nagar {
131dbf3da1cSFrederic Weisbecker u64 utime, stime, stimescaled, utimescaled;
13268f6783dSThomas Gleixner unsigned long long t2, t3;
13368f6783dSThomas Gleixner unsigned long flags, t1;
13468f6783dSThomas Gleixner s64 tmp;
1356f44993fSShailabh Nagar
136dbf3da1cSFrederic Weisbecker task_cputime(tsk, &utime, &stime);
13768f6783dSThomas Gleixner tmp = (s64)d->cpu_run_real_total;
138dbf3da1cSFrederic Weisbecker tmp += utime + stime;
1396f44993fSShailabh Nagar d->cpu_run_real_total = (tmp < (s64)d->cpu_run_real_total) ? 0 : tmp;
1406f44993fSShailabh Nagar
141dbf3da1cSFrederic Weisbecker task_cputime_scaled(tsk, &utimescaled, &stimescaled);
14268f6783dSThomas Gleixner tmp = (s64)d->cpu_scaled_run_real_total;
143dbf3da1cSFrederic Weisbecker tmp += utimescaled + stimescaled;
144c66f08beSMichael Neuling d->cpu_scaled_run_real_total =
145c66f08beSMichael Neuling (tmp < (s64)d->cpu_scaled_run_real_total) ? 0 : tmp;
146c66f08beSMichael Neuling
1476f44993fSShailabh Nagar /*
1486f44993fSShailabh Nagar * No locking available for sched_info (and too expensive to add one)
1496f44993fSShailabh Nagar * Mitigate by taking snapshot of values
1506f44993fSShailabh Nagar */
1512d72376bSIngo Molnar t1 = tsk->sched_info.pcount;
1526f44993fSShailabh Nagar t2 = tsk->sched_info.run_delay;
1539c2c4802SKen Chen t3 = tsk->se.sum_exec_runtime;
1546f44993fSShailabh Nagar
1556f44993fSShailabh Nagar d->cpu_count += t1;
1566f44993fSShailabh Nagar
157172ba844SBalbir Singh tmp = (s64)d->cpu_delay_total + t2;
1586f44993fSShailabh Nagar d->cpu_delay_total = (tmp < (s64)d->cpu_delay_total) ? 0 : tmp;
1596f44993fSShailabh Nagar
160172ba844SBalbir Singh tmp = (s64)d->cpu_run_virtual_total + t3;
1616f44993fSShailabh Nagar d->cpu_run_virtual_total =
1626f44993fSShailabh Nagar (tmp < (s64)d->cpu_run_virtual_total) ? 0 : tmp;
1636f44993fSShailabh Nagar
164e4042ad4SPeter Zijlstra if (!tsk->delays)
165e4042ad4SPeter Zijlstra return 0;
166e4042ad4SPeter Zijlstra
1676f44993fSShailabh Nagar /* zero XXX_total, non-zero XXX_count implies XXX stat overflowed */
1686f44993fSShailabh Nagar
16902acc80dSSebastian Andrzej Siewior raw_spin_lock_irqsave(&tsk->delays->lock, flags);
1706f44993fSShailabh Nagar tmp = d->blkio_delay_total + tsk->delays->blkio_delay;
1716f44993fSShailabh Nagar d->blkio_delay_total = (tmp < d->blkio_delay_total) ? 0 : tmp;
1726f44993fSShailabh Nagar tmp = d->swapin_delay_total + tsk->delays->swapin_delay;
1736f44993fSShailabh Nagar d->swapin_delay_total = (tmp < d->swapin_delay_total) ? 0 : tmp;
174016ae219SKeika Kobayashi tmp = d->freepages_delay_total + tsk->delays->freepages_delay;
175016ae219SKeika Kobayashi d->freepages_delay_total = (tmp < d->freepages_delay_total) ? 0 : tmp;
176b1d29ba8SJohannes Weiner tmp = d->thrashing_delay_total + tsk->delays->thrashing_delay;
177b1d29ba8SJohannes Weiner d->thrashing_delay_total = (tmp < d->thrashing_delay_total) ? 0 : tmp;
1785bf18281Swangyong tmp = d->compact_delay_total + tsk->delays->compact_delay;
1795bf18281Swangyong d->compact_delay_total = (tmp < d->compact_delay_total) ? 0 : tmp;
180662ce1dcSYang Yang tmp = d->wpcopy_delay_total + tsk->delays->wpcopy_delay;
181662ce1dcSYang Yang d->wpcopy_delay_total = (tmp < d->wpcopy_delay_total) ? 0 : tmp;
182*a3b2aeacSYang Yang tmp = d->irq_delay_total + tsk->delays->irq_delay;
183*a3b2aeacSYang Yang d->irq_delay_total = (tmp < d->irq_delay_total) ? 0 : tmp;
1846f44993fSShailabh Nagar d->blkio_count += tsk->delays->blkio_count;
1856f44993fSShailabh Nagar d->swapin_count += tsk->delays->swapin_count;
186016ae219SKeika Kobayashi d->freepages_count += tsk->delays->freepages_count;
187b1d29ba8SJohannes Weiner d->thrashing_count += tsk->delays->thrashing_count;
1885bf18281Swangyong d->compact_count += tsk->delays->compact_count;
189662ce1dcSYang Yang d->wpcopy_count += tsk->delays->wpcopy_count;
190*a3b2aeacSYang Yang d->irq_count += tsk->delays->irq_count;
19102acc80dSSebastian Andrzej Siewior raw_spin_unlock_irqrestore(&tsk->delays->lock, flags);
1926f44993fSShailabh Nagar
1936f44993fSShailabh Nagar return 0;
1946f44993fSShailabh Nagar }
19525890454SShailabh Nagar
__delayacct_blkio_ticks(struct task_struct * tsk)19625890454SShailabh Nagar __u64 __delayacct_blkio_ticks(struct task_struct *tsk)
19725890454SShailabh Nagar {
19825890454SShailabh Nagar __u64 ret;
19964efade1SPeter Zijlstra unsigned long flags;
20025890454SShailabh Nagar
20102acc80dSSebastian Andrzej Siewior raw_spin_lock_irqsave(&tsk->delays->lock, flags);
202a3d5dc90SYang Yang ret = nsec_to_clock_t(tsk->delays->blkio_delay);
20302acc80dSSebastian Andrzej Siewior raw_spin_unlock_irqrestore(&tsk->delays->lock, flags);
20425890454SShailabh Nagar return ret;
20525890454SShailabh Nagar }
20625890454SShailabh Nagar
__delayacct_freepages_start(void)207873b4771SKeika Kobayashi void __delayacct_freepages_start(void)
208873b4771SKeika Kobayashi {
2094b7a08a0SPeter Zijlstra current->delays->freepages_start = local_clock();
210873b4771SKeika Kobayashi }
211873b4771SKeika Kobayashi
__delayacct_freepages_end(void)212873b4771SKeika Kobayashi void __delayacct_freepages_end(void)
213873b4771SKeika Kobayashi {
2144b7a08a0SPeter Zijlstra delayacct_end(¤t->delays->lock,
215c96f5471SJosh Snyder ¤t->delays->freepages_start,
216873b4771SKeika Kobayashi ¤t->delays->freepages_delay,
217873b4771SKeika Kobayashi ¤t->delays->freepages_count);
218873b4771SKeika Kobayashi }
219873b4771SKeika Kobayashi
__delayacct_thrashing_start(bool * in_thrashing)220aa1cf99bSYang Yang void __delayacct_thrashing_start(bool *in_thrashing)
221b1d29ba8SJohannes Weiner {
222aa1cf99bSYang Yang *in_thrashing = !!current->in_thrashing;
223aa1cf99bSYang Yang if (*in_thrashing)
224aa1cf99bSYang Yang return;
225aa1cf99bSYang Yang
226aa1cf99bSYang Yang current->in_thrashing = 1;
2274b7a08a0SPeter Zijlstra current->delays->thrashing_start = local_clock();
228b1d29ba8SJohannes Weiner }
229b1d29ba8SJohannes Weiner
__delayacct_thrashing_end(bool * in_thrashing)230aa1cf99bSYang Yang void __delayacct_thrashing_end(bool *in_thrashing)
231b1d29ba8SJohannes Weiner {
232aa1cf99bSYang Yang if (*in_thrashing)
233aa1cf99bSYang Yang return;
234aa1cf99bSYang Yang
235aa1cf99bSYang Yang current->in_thrashing = 0;
236b1d29ba8SJohannes Weiner delayacct_end(¤t->delays->lock,
237b1d29ba8SJohannes Weiner ¤t->delays->thrashing_start,
238b1d29ba8SJohannes Weiner ¤t->delays->thrashing_delay,
239b1d29ba8SJohannes Weiner ¤t->delays->thrashing_count);
240b1d29ba8SJohannes Weiner }
241a3d5dc90SYang Yang
__delayacct_swapin_start(void)242a3d5dc90SYang Yang void __delayacct_swapin_start(void)
243a3d5dc90SYang Yang {
244a3d5dc90SYang Yang current->delays->swapin_start = local_clock();
245a3d5dc90SYang Yang }
246a3d5dc90SYang Yang
__delayacct_swapin_end(void)247a3d5dc90SYang Yang void __delayacct_swapin_end(void)
248a3d5dc90SYang Yang {
249a3d5dc90SYang Yang delayacct_end(¤t->delays->lock,
250a3d5dc90SYang Yang ¤t->delays->swapin_start,
251a3d5dc90SYang Yang ¤t->delays->swapin_delay,
252a3d5dc90SYang Yang ¤t->delays->swapin_count);
253a3d5dc90SYang Yang }
2545bf18281Swangyong
__delayacct_compact_start(void)2555bf18281Swangyong void __delayacct_compact_start(void)
2565bf18281Swangyong {
2575bf18281Swangyong current->delays->compact_start = local_clock();
2585bf18281Swangyong }
2595bf18281Swangyong
__delayacct_compact_end(void)2605bf18281Swangyong void __delayacct_compact_end(void)
2615bf18281Swangyong {
2625bf18281Swangyong delayacct_end(¤t->delays->lock,
2635bf18281Swangyong ¤t->delays->compact_start,
2645bf18281Swangyong ¤t->delays->compact_delay,
2655bf18281Swangyong ¤t->delays->compact_count);
2665bf18281Swangyong }
267662ce1dcSYang Yang
__delayacct_wpcopy_start(void)268662ce1dcSYang Yang void __delayacct_wpcopy_start(void)
269662ce1dcSYang Yang {
270662ce1dcSYang Yang current->delays->wpcopy_start = local_clock();
271662ce1dcSYang Yang }
272662ce1dcSYang Yang
__delayacct_wpcopy_end(void)273662ce1dcSYang Yang void __delayacct_wpcopy_end(void)
274662ce1dcSYang Yang {
275662ce1dcSYang Yang delayacct_end(¤t->delays->lock,
276662ce1dcSYang Yang ¤t->delays->wpcopy_start,
277662ce1dcSYang Yang ¤t->delays->wpcopy_delay,
278662ce1dcSYang Yang ¤t->delays->wpcopy_count);
279662ce1dcSYang Yang }
280*a3b2aeacSYang Yang
__delayacct_irq(struct task_struct * task,u32 delta)281*a3b2aeacSYang Yang void __delayacct_irq(struct task_struct *task, u32 delta)
282*a3b2aeacSYang Yang {
283*a3b2aeacSYang Yang unsigned long flags;
284*a3b2aeacSYang Yang
285*a3b2aeacSYang Yang raw_spin_lock_irqsave(&task->delays->lock, flags);
286*a3b2aeacSYang Yang task->delays->irq_delay += delta;
287*a3b2aeacSYang Yang task->delays->irq_count++;
288*a3b2aeacSYang Yang raw_spin_unlock_irqrestore(&task->delays->lock, flags);
289*a3b2aeacSYang Yang }
290*a3b2aeacSYang Yang
291