1 /* 2 * sched_clock.c: support for extending counters to full 64-bit ns counter 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 #include <linux/clocksource.h> 9 #include <linux/init.h> 10 #include <linux/jiffies.h> 11 #include <linux/ktime.h> 12 #include <linux/kernel.h> 13 #include <linux/moduleparam.h> 14 #include <linux/sched.h> 15 #include <linux/syscore_ops.h> 16 #include <linux/hrtimer.h> 17 #include <linux/sched_clock.h> 18 #include <linux/seqlock.h> 19 #include <linux/bitops.h> 20 21 struct clock_data { 22 ktime_t wrap_kt; 23 u64 epoch_ns; 24 u64 epoch_cyc; 25 seqcount_t seq; 26 unsigned long rate; 27 u32 mult; 28 u32 shift; 29 bool suspended; 30 }; 31 32 static struct hrtimer sched_clock_timer; 33 static int irqtime = -1; 34 35 core_param(irqtime, irqtime, int, 0400); 36 37 static struct clock_data cd = { 38 .mult = NSEC_PER_SEC / HZ, 39 }; 40 41 static u64 __read_mostly sched_clock_mask; 42 43 static u64 notrace jiffy_sched_clock_read(void) 44 { 45 /* 46 * We don't need to use get_jiffies_64 on 32-bit arches here 47 * because we register with BITS_PER_LONG 48 */ 49 return (u64)(jiffies - INITIAL_JIFFIES); 50 } 51 52 static u32 __read_mostly (*read_sched_clock_32)(void); 53 54 static u64 notrace read_sched_clock_32_wrapper(void) 55 { 56 return read_sched_clock_32(); 57 } 58 59 static u64 __read_mostly (*read_sched_clock)(void) = jiffy_sched_clock_read; 60 61 static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift) 62 { 63 return (cyc * mult) >> shift; 64 } 65 66 unsigned long long notrace sched_clock(void) 67 { 68 u64 epoch_ns; 69 u64 epoch_cyc; 70 u64 cyc; 71 unsigned long seq; 72 73 if (cd.suspended) 74 return cd.epoch_ns; 75 76 do { 77 seq = raw_read_seqcount_begin(&cd.seq); 78 epoch_cyc = cd.epoch_cyc; 79 epoch_ns = cd.epoch_ns; 80 } while (read_seqcount_retry(&cd.seq, seq)); 81 82 cyc = read_sched_clock(); 83 cyc = (cyc - epoch_cyc) & sched_clock_mask; 84 return epoch_ns + cyc_to_ns(cyc, cd.mult, cd.shift); 85 } 86 87 /* 88 * Atomically update the sched_clock epoch. 89 */ 90 static void notrace update_sched_clock(void) 91 { 92 unsigned long flags; 93 u64 cyc; 94 u64 ns; 95 96 cyc = read_sched_clock(); 97 ns = cd.epoch_ns + 98 cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask, 99 cd.mult, cd.shift); 100 101 raw_local_irq_save(flags); 102 raw_write_seqcount_begin(&cd.seq); 103 cd.epoch_ns = ns; 104 cd.epoch_cyc = cyc; 105 raw_write_seqcount_end(&cd.seq); 106 raw_local_irq_restore(flags); 107 } 108 109 static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt) 110 { 111 update_sched_clock(); 112 hrtimer_forward_now(hrt, cd.wrap_kt); 113 return HRTIMER_RESTART; 114 } 115 116 void __init sched_clock_register(u64 (*read)(void), int bits, 117 unsigned long rate) 118 { 119 u64 res, wrap, new_mask, new_epoch, cyc, ns; 120 u32 new_mult, new_shift; 121 ktime_t new_wrap_kt; 122 unsigned long r; 123 char r_unit; 124 125 if (cd.rate > rate) 126 return; 127 128 WARN_ON(!irqs_disabled()); 129 130 /* calculate the mult/shift to convert counter ticks to ns. */ 131 clocks_calc_mult_shift(&new_mult, &new_shift, rate, NSEC_PER_SEC, 3600); 132 133 new_mask = CLOCKSOURCE_MASK(bits); 134 135 /* calculate how many ns until we wrap */ 136 wrap = clocks_calc_max_nsecs(new_mult, new_shift, 0, new_mask); 137 new_wrap_kt = ns_to_ktime(wrap - (wrap >> 3)); 138 139 /* update epoch for new counter and update epoch_ns from old counter*/ 140 new_epoch = read(); 141 cyc = read_sched_clock(); 142 ns = cd.epoch_ns + cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask, 143 cd.mult, cd.shift); 144 145 raw_write_seqcount_begin(&cd.seq); 146 read_sched_clock = read; 147 sched_clock_mask = new_mask; 148 cd.rate = rate; 149 cd.wrap_kt = new_wrap_kt; 150 cd.mult = new_mult; 151 cd.shift = new_shift; 152 cd.epoch_cyc = new_epoch; 153 cd.epoch_ns = ns; 154 raw_write_seqcount_end(&cd.seq); 155 156 r = rate; 157 if (r >= 4000000) { 158 r /= 1000000; 159 r_unit = 'M'; 160 } else if (r >= 1000) { 161 r /= 1000; 162 r_unit = 'k'; 163 } else 164 r_unit = ' '; 165 166 /* calculate the ns resolution of this counter */ 167 res = cyc_to_ns(1ULL, new_mult, new_shift); 168 169 pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n", 170 bits, r, r_unit, res, wrap); 171 172 /* Enable IRQ time accounting if we have a fast enough sched_clock */ 173 if (irqtime > 0 || (irqtime == -1 && rate >= 1000000)) 174 enable_sched_clock_irqtime(); 175 176 pr_debug("Registered %pF as sched_clock source\n", read); 177 } 178 179 void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate) 180 { 181 read_sched_clock_32 = read; 182 sched_clock_register(read_sched_clock_32_wrapper, bits, rate); 183 } 184 185 void __init sched_clock_postinit(void) 186 { 187 /* 188 * If no sched_clock function has been provided at that point, 189 * make it the final one one. 190 */ 191 if (read_sched_clock == jiffy_sched_clock_read) 192 sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ); 193 194 update_sched_clock(); 195 196 /* 197 * Start the timer to keep sched_clock() properly updated and 198 * sets the initial epoch. 199 */ 200 hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 201 sched_clock_timer.function = sched_clock_poll; 202 hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); 203 } 204 205 static int sched_clock_suspend(void) 206 { 207 sched_clock_poll(&sched_clock_timer); 208 cd.suspended = true; 209 return 0; 210 } 211 212 static void sched_clock_resume(void) 213 { 214 cd.epoch_cyc = read_sched_clock(); 215 cd.suspended = false; 216 } 217 218 static struct syscore_ops sched_clock_ops = { 219 .suspend = sched_clock_suspend, 220 .resume = sched_clock_resume, 221 }; 222 223 static int __init sched_clock_syscore_init(void) 224 { 225 register_syscore_ops(&sched_clock_ops); 226 return 0; 227 } 228 device_initcall(sched_clock_syscore_init); 229