135728b82SThomas Gleixner // SPDX-License-Identifier: GPL-2.0
238ff87f7SStephen Boyd /*
358c5fc2bSThomas Gleixner * Generic sched_clock() support, to extend low level hardware time
458c5fc2bSThomas Gleixner * counters to full 64-bit ns values.
538ff87f7SStephen Boyd */
638ff87f7SStephen Boyd #include <linux/clocksource.h>
738ff87f7SStephen Boyd #include <linux/init.h>
838ff87f7SStephen Boyd #include <linux/jiffies.h>
9a08ca5d1SStephen Boyd #include <linux/ktime.h>
1038ff87f7SStephen Boyd #include <linux/kernel.h>
1192067440SMaciej W. Rozycki #include <linux/math.h>
1238ff87f7SStephen Boyd #include <linux/moduleparam.h>
1338ff87f7SStephen Boyd #include <linux/sched.h>
14e6017571SIngo Molnar #include <linux/sched/clock.h>
1538ff87f7SStephen Boyd #include <linux/syscore_ops.h>
16a08ca5d1SStephen Boyd #include <linux/hrtimer.h>
1738ff87f7SStephen Boyd #include <linux/sched_clock.h>
1885c3d2ddSStephen Boyd #include <linux/seqlock.h>
19e7e3ff1bSStephen Boyd #include <linux/bitops.h>
2038ff87f7SStephen Boyd
21086ee46bSBen Dooks (Codethink) #include "timekeeping.h"
22086ee46bSBen Dooks (Codethink)
23cf7c9c17SDaniel Thompson /**
2432fea568SIngo Molnar * struct clock_data - all data needed for sched_clock() (including
25cf7c9c17SDaniel Thompson * registration of a new clock source)
26cf7c9c17SDaniel Thompson *
271809bfa4SDaniel Thompson * @seq: Sequence counter for protecting updates. The lowest
281809bfa4SDaniel Thompson * bit is the index for @read_data.
29cf7c9c17SDaniel Thompson * @read_data: Data required to read from sched_clock.
3032fea568SIngo Molnar * @wrap_kt: Duration for which clock can run before wrapping.
3132fea568SIngo Molnar * @rate: Tick rate of the registered clock.
3232fea568SIngo Molnar * @actual_read_sched_clock: Registered hardware level clock read function.
33cf7c9c17SDaniel Thompson *
34cf7c9c17SDaniel Thompson * The ordering of this structure has been chosen to optimize cache
3532fea568SIngo Molnar * performance. In particular 'seq' and 'read_data[0]' (combined) should fit
3632fea568SIngo Molnar * into a single 64-byte cache line.
37cf7c9c17SDaniel Thompson */
38cf7c9c17SDaniel Thompson struct clock_data {
39a690ed07SAhmed S. Darwish seqcount_latch_t seq;
401809bfa4SDaniel Thompson struct clock_read_data read_data[2];
41cf7c9c17SDaniel Thompson ktime_t wrap_kt;
42cf7c9c17SDaniel Thompson unsigned long rate;
4332fea568SIngo Molnar
4413dbeb38SDaniel Thompson u64 (*actual_read_sched_clock)(void);
45cf7c9c17SDaniel Thompson };
46cf7c9c17SDaniel Thompson
47a08ca5d1SStephen Boyd static struct hrtimer sched_clock_timer;
4838ff87f7SStephen Boyd static int irqtime = -1;
4938ff87f7SStephen Boyd
5038ff87f7SStephen Boyd core_param(irqtime, irqtime, int, 0400);
5138ff87f7SStephen Boyd
jiffy_sched_clock_read(void)52e7e3ff1bSStephen Boyd static u64 notrace jiffy_sched_clock_read(void)
5338ff87f7SStephen Boyd {
54e7e3ff1bSStephen Boyd /*
55e7e3ff1bSStephen Boyd * We don't need to use get_jiffies_64 on 32-bit arches here
56e7e3ff1bSStephen Boyd * because we register with BITS_PER_LONG
57e7e3ff1bSStephen Boyd */
58e7e3ff1bSStephen Boyd return (u64)(jiffies - INITIAL_JIFFIES);
5938ff87f7SStephen Boyd }
6038ff87f7SStephen Boyd
61cf7c9c17SDaniel Thompson static struct clock_data cd ____cacheline_aligned = {
621809bfa4SDaniel Thompson .read_data[0] = { .mult = NSEC_PER_SEC / HZ,
63cf7c9c17SDaniel Thompson .read_sched_clock = jiffy_sched_clock_read, },
6413dbeb38SDaniel Thompson .actual_read_sched_clock = jiffy_sched_clock_read,
65cf7c9c17SDaniel Thompson };
6638ff87f7SStephen Boyd
cyc_to_ns(u64 cyc,u32 mult,u32 shift)67*5949a68cSPeter Zijlstra static __always_inline u64 cyc_to_ns(u64 cyc, u32 mult, u32 shift)
6838ff87f7SStephen Boyd {
6938ff87f7SStephen Boyd return (cyc * mult) >> shift;
7038ff87f7SStephen Boyd }
7138ff87f7SStephen Boyd
sched_clock_read_begin(unsigned int * seq)724cd2bb12SQuanyang Wang notrace struct clock_read_data *sched_clock_read_begin(unsigned int *seq)
731b86abc1SPeter Zijlstra {
74aadd6e5cSAhmed S. Darwish *seq = raw_read_seqcount_latch(&cd.seq);
751b86abc1SPeter Zijlstra return cd.read_data + (*seq & 1);
761b86abc1SPeter Zijlstra }
771b86abc1SPeter Zijlstra
sched_clock_read_retry(unsigned int seq)784cd2bb12SQuanyang Wang notrace int sched_clock_read_retry(unsigned int seq)
791b86abc1SPeter Zijlstra {
80d16317deSPeter Zijlstra return raw_read_seqcount_latch_retry(&cd.seq, seq);
811b86abc1SPeter Zijlstra }
821b86abc1SPeter Zijlstra
sched_clock_noinstr(void)83*5949a68cSPeter Zijlstra unsigned long long noinstr sched_clock_noinstr(void)
8438ff87f7SStephen Boyd {
851809bfa4SDaniel Thompson struct clock_read_data *rd;
86*5949a68cSPeter Zijlstra unsigned int seq;
87*5949a68cSPeter Zijlstra u64 cyc, res;
88336ae118SStephen Boyd
8938ff87f7SStephen Boyd do {
90*5949a68cSPeter Zijlstra seq = raw_read_seqcount_latch(&cd.seq);
91*5949a68cSPeter Zijlstra rd = cd.read_data + (seq & 1);
928710e914SDaniel Thompson
9313dbeb38SDaniel Thompson cyc = (rd->read_sched_clock() - rd->epoch_cyc) &
9413dbeb38SDaniel Thompson rd->sched_clock_mask;
9513dbeb38SDaniel Thompson res = rd->epoch_ns + cyc_to_ns(cyc, rd->mult, rd->shift);
96*5949a68cSPeter Zijlstra } while (raw_read_seqcount_latch_retry(&cd.seq, seq));
9738ff87f7SStephen Boyd
988710e914SDaniel Thompson return res;
9938ff87f7SStephen Boyd }
10038ff87f7SStephen Boyd
sched_clock(void)101*5949a68cSPeter Zijlstra unsigned long long notrace sched_clock(void)
102*5949a68cSPeter Zijlstra {
103*5949a68cSPeter Zijlstra unsigned long long ns;
104*5949a68cSPeter Zijlstra preempt_disable_notrace();
105*5949a68cSPeter Zijlstra ns = sched_clock_noinstr();
106*5949a68cSPeter Zijlstra preempt_enable_notrace();
107*5949a68cSPeter Zijlstra return ns;
108*5949a68cSPeter Zijlstra }
109*5949a68cSPeter Zijlstra
11038ff87f7SStephen Boyd /*
1111809bfa4SDaniel Thompson * Updating the data required to read the clock.
1121809bfa4SDaniel Thompson *
11332fea568SIngo Molnar * sched_clock() will never observe mis-matched data even if called from
1141809bfa4SDaniel Thompson * an NMI. We do this by maintaining an odd/even copy of the data and
11532fea568SIngo Molnar * steering sched_clock() to one or the other using a sequence counter.
11632fea568SIngo Molnar * In order to preserve the data cache profile of sched_clock() as much
1171809bfa4SDaniel Thompson * as possible the system reverts back to the even copy when the update
1181809bfa4SDaniel Thompson * completes; the odd copy is used *only* during an update.
1191809bfa4SDaniel Thompson */
update_clock_read_data(struct clock_read_data * rd)1201809bfa4SDaniel Thompson static void update_clock_read_data(struct clock_read_data *rd)
1211809bfa4SDaniel Thompson {
1221809bfa4SDaniel Thompson /* update the backup (odd) copy with the new data */
1231809bfa4SDaniel Thompson cd.read_data[1] = *rd;
1241809bfa4SDaniel Thompson
1251809bfa4SDaniel Thompson /* steer readers towards the odd copy */
1261809bfa4SDaniel Thompson raw_write_seqcount_latch(&cd.seq);
1271809bfa4SDaniel Thompson
1281809bfa4SDaniel Thompson /* now its safe for us to update the normal (even) copy */
1291809bfa4SDaniel Thompson cd.read_data[0] = *rd;
1301809bfa4SDaniel Thompson
1311809bfa4SDaniel Thompson /* switch readers back to the even copy */
1321809bfa4SDaniel Thompson raw_write_seqcount_latch(&cd.seq);
1331809bfa4SDaniel Thompson }
1341809bfa4SDaniel Thompson
1351809bfa4SDaniel Thompson /*
13632fea568SIngo Molnar * Atomically update the sched_clock() epoch.
13738ff87f7SStephen Boyd */
update_sched_clock(void)1389fee69a8SDaniel Thompson static void update_sched_clock(void)
13938ff87f7SStephen Boyd {
140e7e3ff1bSStephen Boyd u64 cyc;
14138ff87f7SStephen Boyd u64 ns;
1421809bfa4SDaniel Thompson struct clock_read_data rd;
1431809bfa4SDaniel Thompson
1441809bfa4SDaniel Thompson rd = cd.read_data[0];
14538ff87f7SStephen Boyd
14613dbeb38SDaniel Thompson cyc = cd.actual_read_sched_clock();
14732fea568SIngo Molnar ns = rd.epoch_ns + cyc_to_ns((cyc - rd.epoch_cyc) & rd.sched_clock_mask, rd.mult, rd.shift);
14885c3d2ddSStephen Boyd
1491809bfa4SDaniel Thompson rd.epoch_ns = ns;
1501809bfa4SDaniel Thompson rd.epoch_cyc = cyc;
1511809bfa4SDaniel Thompson
1521809bfa4SDaniel Thompson update_clock_read_data(&rd);
15338ff87f7SStephen Boyd }
15438ff87f7SStephen Boyd
sched_clock_poll(struct hrtimer * hrt)155a08ca5d1SStephen Boyd static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt)
15638ff87f7SStephen Boyd {
15738ff87f7SStephen Boyd update_sched_clock();
158a08ca5d1SStephen Boyd hrtimer_forward_now(hrt, cd.wrap_kt);
15932fea568SIngo Molnar
160a08ca5d1SStephen Boyd return HRTIMER_RESTART;
16138ff87f7SStephen Boyd }
16238ff87f7SStephen Boyd
16332fea568SIngo Molnar void __init
sched_clock_register(u64 (* read)(void),int bits,unsigned long rate)16432fea568SIngo Molnar sched_clock_register(u64 (*read)(void), int bits, unsigned long rate)
16538ff87f7SStephen Boyd {
1665ae8aabeSStephen Boyd u64 res, wrap, new_mask, new_epoch, cyc, ns;
1675ae8aabeSStephen Boyd u32 new_mult, new_shift;
16827077455SPaul Cercueil unsigned long r, flags;
16938ff87f7SStephen Boyd char r_unit;
1701809bfa4SDaniel Thompson struct clock_read_data rd;
17138ff87f7SStephen Boyd
17238ff87f7SStephen Boyd if (cd.rate > rate)
17338ff87f7SStephen Boyd return;
17438ff87f7SStephen Boyd
17527077455SPaul Cercueil /* Cannot register a sched_clock with interrupts on */
17627077455SPaul Cercueil local_irq_save(flags);
17738ff87f7SStephen Boyd
17832fea568SIngo Molnar /* Calculate the mult/shift to convert counter ticks to ns. */
1795ae8aabeSStephen Boyd clocks_calc_mult_shift(&new_mult, &new_shift, rate, NSEC_PER_SEC, 3600);
1805ae8aabeSStephen Boyd
1815ae8aabeSStephen Boyd new_mask = CLOCKSOURCE_MASK(bits);
1828710e914SDaniel Thompson cd.rate = rate;
1835ae8aabeSStephen Boyd
18432fea568SIngo Molnar /* Calculate how many nanosecs until we risk wrapping */
185fb82fe2fSJohn Stultz wrap = clocks_calc_max_nsecs(new_mult, new_shift, 0, new_mask, NULL);
1868710e914SDaniel Thompson cd.wrap_kt = ns_to_ktime(wrap);
1875ae8aabeSStephen Boyd
1881809bfa4SDaniel Thompson rd = cd.read_data[0];
1891809bfa4SDaniel Thompson
19032fea568SIngo Molnar /* Update epoch for new counter and update 'epoch_ns' from old counter*/
1915ae8aabeSStephen Boyd new_epoch = read();
19213dbeb38SDaniel Thompson cyc = cd.actual_read_sched_clock();
19332fea568SIngo Molnar ns = rd.epoch_ns + cyc_to_ns((cyc - rd.epoch_cyc) & rd.sched_clock_mask, rd.mult, rd.shift);
19413dbeb38SDaniel Thompson cd.actual_read_sched_clock = read;
1955ae8aabeSStephen Boyd
1961809bfa4SDaniel Thompson rd.read_sched_clock = read;
1971809bfa4SDaniel Thompson rd.sched_clock_mask = new_mask;
1981809bfa4SDaniel Thompson rd.mult = new_mult;
1991809bfa4SDaniel Thompson rd.shift = new_shift;
2001809bfa4SDaniel Thompson rd.epoch_cyc = new_epoch;
2011809bfa4SDaniel Thompson rd.epoch_ns = ns;
20232fea568SIngo Molnar
2031809bfa4SDaniel Thompson update_clock_read_data(&rd);
20438ff87f7SStephen Boyd
2051b8955bcSDavid Engraf if (sched_clock_timer.function != NULL) {
2061b8955bcSDavid Engraf /* update timeout for clock wrap */
2072c8bd588SAhmed S. Darwish hrtimer_start(&sched_clock_timer, cd.wrap_kt,
2082c8bd588SAhmed S. Darwish HRTIMER_MODE_REL_HARD);
2091b8955bcSDavid Engraf }
2101b8955bcSDavid Engraf
21138ff87f7SStephen Boyd r = rate;
21238ff87f7SStephen Boyd if (r >= 4000000) {
21392067440SMaciej W. Rozycki r = DIV_ROUND_CLOSEST(r, 1000000);
21438ff87f7SStephen Boyd r_unit = 'M';
215f4b62e1eSMaciej W. Rozycki } else if (r >= 4000) {
21692067440SMaciej W. Rozycki r = DIV_ROUND_CLOSEST(r, 1000);
21738ff87f7SStephen Boyd r_unit = 'k';
21832fea568SIngo Molnar } else {
21938ff87f7SStephen Boyd r_unit = ' ';
22032fea568SIngo Molnar }
22138ff87f7SStephen Boyd
22232fea568SIngo Molnar /* Calculate the ns resolution of this counter */
2235ae8aabeSStephen Boyd res = cyc_to_ns(1ULL, new_mult, new_shift);
2245ae8aabeSStephen Boyd
225a08ca5d1SStephen Boyd pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n",
226a08ca5d1SStephen Boyd bits, r, r_unit, res, wrap);
22738ff87f7SStephen Boyd
22832fea568SIngo Molnar /* Enable IRQ time accounting if we have a fast enough sched_clock() */
22938ff87f7SStephen Boyd if (irqtime > 0 || (irqtime == -1 && rate >= 1000000))
23038ff87f7SStephen Boyd enable_sched_clock_irqtime();
23138ff87f7SStephen Boyd
23227077455SPaul Cercueil local_irq_restore(flags);
23327077455SPaul Cercueil
234d75f773cSSakari Ailus pr_debug("Registered %pS as sched_clock source\n", read);
23538ff87f7SStephen Boyd }
23638ff87f7SStephen Boyd
generic_sched_clock_init(void)2375d2a4e91SPavel Tatashin void __init generic_sched_clock_init(void)
23838ff87f7SStephen Boyd {
23938ff87f7SStephen Boyd /*
24032fea568SIngo Molnar * If no sched_clock() function has been provided at that point,
241b0294f30SRandy Dunlap * make it the final one.
24238ff87f7SStephen Boyd */
24313dbeb38SDaniel Thompson if (cd.actual_read_sched_clock == jiffy_sched_clock_read)
244e7e3ff1bSStephen Boyd sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ);
24538ff87f7SStephen Boyd
246a08ca5d1SStephen Boyd update_sched_clock();
247a08ca5d1SStephen Boyd
248a08ca5d1SStephen Boyd /*
249a08ca5d1SStephen Boyd * Start the timer to keep sched_clock() properly updated and
250a08ca5d1SStephen Boyd * sets the initial epoch.
251a08ca5d1SStephen Boyd */
2522c8bd588SAhmed S. Darwish hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
253a08ca5d1SStephen Boyd sched_clock_timer.function = sched_clock_poll;
2542c8bd588SAhmed S. Darwish hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL_HARD);
25538ff87f7SStephen Boyd }
25638ff87f7SStephen Boyd
25713dbeb38SDaniel Thompson /*
25813dbeb38SDaniel Thompson * Clock read function for use when the clock is suspended.
25913dbeb38SDaniel Thompson *
26013dbeb38SDaniel Thompson * This function makes it appear to sched_clock() as if the clock
26113dbeb38SDaniel Thompson * stopped counting at its last update.
2621809bfa4SDaniel Thompson *
2631809bfa4SDaniel Thompson * This function must only be called from the critical
2641809bfa4SDaniel Thompson * section in sched_clock(). It relies on the read_seqcount_retry()
2651809bfa4SDaniel Thompson * at the end of the critical section to be sure we observe the
26632fea568SIngo Molnar * correct copy of 'epoch_cyc'.
26713dbeb38SDaniel Thompson */
suspended_sched_clock_read(void)26813dbeb38SDaniel Thompson static u64 notrace suspended_sched_clock_read(void)
26913dbeb38SDaniel Thompson {
27058faf20aSAhmed S. Darwish unsigned int seq = raw_read_seqcount_latch(&cd.seq);
2711809bfa4SDaniel Thompson
2721809bfa4SDaniel Thompson return cd.read_data[seq & 1].epoch_cyc;
27313dbeb38SDaniel Thompson }
27413dbeb38SDaniel Thompson
sched_clock_suspend(void)2753f2552f7SChang-An Chen int sched_clock_suspend(void)
27638ff87f7SStephen Boyd {
2771809bfa4SDaniel Thompson struct clock_read_data *rd = &cd.read_data[0];
278cf7c9c17SDaniel Thompson
279f723aa18SStephen Boyd update_sched_clock();
280f723aa18SStephen Boyd hrtimer_cancel(&sched_clock_timer);
28113dbeb38SDaniel Thompson rd->read_sched_clock = suspended_sched_clock_read;
28232fea568SIngo Molnar
28338ff87f7SStephen Boyd return 0;
28438ff87f7SStephen Boyd }
28538ff87f7SStephen Boyd
sched_clock_resume(void)2863f2552f7SChang-An Chen void sched_clock_resume(void)
28738ff87f7SStephen Boyd {
2881809bfa4SDaniel Thompson struct clock_read_data *rd = &cd.read_data[0];
289cf7c9c17SDaniel Thompson
29013dbeb38SDaniel Thompson rd->epoch_cyc = cd.actual_read_sched_clock();
2912c8bd588SAhmed S. Darwish hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL_HARD);
29213dbeb38SDaniel Thompson rd->read_sched_clock = cd.actual_read_sched_clock;
29338ff87f7SStephen Boyd }
29438ff87f7SStephen Boyd
29538ff87f7SStephen Boyd static struct syscore_ops sched_clock_ops = {
29638ff87f7SStephen Boyd .suspend = sched_clock_suspend,
29738ff87f7SStephen Boyd .resume = sched_clock_resume,
29838ff87f7SStephen Boyd };
29938ff87f7SStephen Boyd
sched_clock_syscore_init(void)30038ff87f7SStephen Boyd static int __init sched_clock_syscore_init(void)
30138ff87f7SStephen Boyd {
30238ff87f7SStephen Boyd register_syscore_ops(&sched_clock_ops);
30332fea568SIngo Molnar
30438ff87f7SStephen Boyd return 0;
30538ff87f7SStephen Boyd }
30638ff87f7SStephen Boyd device_initcall(sched_clock_syscore_init);
307