1 /* 2 * sched_clock.c: support for extending counters to full 64-bit ns counter 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 #include <linux/clocksource.h> 9 #include <linux/init.h> 10 #include <linux/jiffies.h> 11 #include <linux/ktime.h> 12 #include <linux/kernel.h> 13 #include <linux/moduleparam.h> 14 #include <linux/sched.h> 15 #include <linux/syscore_ops.h> 16 #include <linux/hrtimer.h> 17 #include <linux/sched_clock.h> 18 #include <linux/seqlock.h> 19 #include <linux/bitops.h> 20 21 struct clock_data { 22 ktime_t wrap_kt; 23 u64 epoch_ns; 24 u64 epoch_cyc; 25 seqcount_t seq; 26 unsigned long rate; 27 u32 mult; 28 u32 shift; 29 bool suspended; 30 }; 31 32 static struct hrtimer sched_clock_timer; 33 static int irqtime = -1; 34 35 core_param(irqtime, irqtime, int, 0400); 36 37 static struct clock_data cd = { 38 .mult = NSEC_PER_SEC / HZ, 39 }; 40 41 static u64 __read_mostly sched_clock_mask; 42 43 static u64 notrace jiffy_sched_clock_read(void) 44 { 45 /* 46 * We don't need to use get_jiffies_64 on 32-bit arches here 47 * because we register with BITS_PER_LONG 48 */ 49 return (u64)(jiffies - INITIAL_JIFFIES); 50 } 51 52 static u64 __read_mostly (*read_sched_clock)(void) = jiffy_sched_clock_read; 53 54 static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift) 55 { 56 return (cyc * mult) >> shift; 57 } 58 59 unsigned long long notrace sched_clock(void) 60 { 61 u64 epoch_ns; 62 u64 epoch_cyc; 63 u64 cyc; 64 unsigned long seq; 65 66 if (cd.suspended) 67 return cd.epoch_ns; 68 69 do { 70 seq = raw_read_seqcount_begin(&cd.seq); 71 epoch_cyc = cd.epoch_cyc; 72 epoch_ns = cd.epoch_ns; 73 } while (read_seqcount_retry(&cd.seq, seq)); 74 75 cyc = read_sched_clock(); 76 cyc = (cyc - epoch_cyc) & sched_clock_mask; 77 return epoch_ns + cyc_to_ns(cyc, cd.mult, cd.shift); 78 } 79 80 /* 81 * Atomically update the sched_clock epoch. 82 */ 83 static void notrace update_sched_clock(void) 84 { 85 unsigned long flags; 86 u64 cyc; 87 u64 ns; 88 89 cyc = read_sched_clock(); 90 ns = cd.epoch_ns + 91 cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask, 92 cd.mult, cd.shift); 93 94 raw_local_irq_save(flags); 95 raw_write_seqcount_begin(&cd.seq); 96 cd.epoch_ns = ns; 97 cd.epoch_cyc = cyc; 98 raw_write_seqcount_end(&cd.seq); 99 raw_local_irq_restore(flags); 100 } 101 102 static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt) 103 { 104 update_sched_clock(); 105 hrtimer_forward_now(hrt, cd.wrap_kt); 106 return HRTIMER_RESTART; 107 } 108 109 void __init sched_clock_register(u64 (*read)(void), int bits, 110 unsigned long rate) 111 { 112 u64 res, wrap, new_mask, new_epoch, cyc, ns; 113 u32 new_mult, new_shift; 114 ktime_t new_wrap_kt; 115 unsigned long r; 116 char r_unit; 117 118 if (cd.rate > rate) 119 return; 120 121 WARN_ON(!irqs_disabled()); 122 123 /* calculate the mult/shift to convert counter ticks to ns. */ 124 clocks_calc_mult_shift(&new_mult, &new_shift, rate, NSEC_PER_SEC, 3600); 125 126 new_mask = CLOCKSOURCE_MASK(bits); 127 128 /* calculate how many ns until we wrap */ 129 wrap = clocks_calc_max_nsecs(new_mult, new_shift, 0, new_mask); 130 new_wrap_kt = ns_to_ktime(wrap - (wrap >> 3)); 131 132 /* update epoch for new counter and update epoch_ns from old counter*/ 133 new_epoch = read(); 134 cyc = read_sched_clock(); 135 ns = cd.epoch_ns + cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask, 136 cd.mult, cd.shift); 137 138 raw_write_seqcount_begin(&cd.seq); 139 read_sched_clock = read; 140 sched_clock_mask = new_mask; 141 cd.rate = rate; 142 cd.wrap_kt = new_wrap_kt; 143 cd.mult = new_mult; 144 cd.shift = new_shift; 145 cd.epoch_cyc = new_epoch; 146 cd.epoch_ns = ns; 147 raw_write_seqcount_end(&cd.seq); 148 149 r = rate; 150 if (r >= 4000000) { 151 r /= 1000000; 152 r_unit = 'M'; 153 } else if (r >= 1000) { 154 r /= 1000; 155 r_unit = 'k'; 156 } else 157 r_unit = ' '; 158 159 /* calculate the ns resolution of this counter */ 160 res = cyc_to_ns(1ULL, new_mult, new_shift); 161 162 pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n", 163 bits, r, r_unit, res, wrap); 164 165 /* Enable IRQ time accounting if we have a fast enough sched_clock */ 166 if (irqtime > 0 || (irqtime == -1 && rate >= 1000000)) 167 enable_sched_clock_irqtime(); 168 169 pr_debug("Registered %pF as sched_clock source\n", read); 170 } 171 172 void __init sched_clock_postinit(void) 173 { 174 /* 175 * If no sched_clock function has been provided at that point, 176 * make it the final one one. 177 */ 178 if (read_sched_clock == jiffy_sched_clock_read) 179 sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ); 180 181 update_sched_clock(); 182 183 /* 184 * Start the timer to keep sched_clock() properly updated and 185 * sets the initial epoch. 186 */ 187 hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 188 sched_clock_timer.function = sched_clock_poll; 189 hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); 190 } 191 192 static int sched_clock_suspend(void) 193 { 194 sched_clock_poll(&sched_clock_timer); 195 cd.suspended = true; 196 return 0; 197 } 198 199 static void sched_clock_resume(void) 200 { 201 cd.epoch_cyc = read_sched_clock(); 202 cd.suspended = false; 203 } 204 205 static struct syscore_ops sched_clock_ops = { 206 .suspend = sched_clock_suspend, 207 .resume = sched_clock_resume, 208 }; 209 210 static int __init sched_clock_syscore_init(void) 211 { 212 register_syscore_ops(&sched_clock_ops); 213 return 0; 214 } 215 device_initcall(sched_clock_syscore_init); 216