135728b82SThomas Gleixner // SPDX-License-Identifier: GPL-2.0+
2734efb46Sjohn stultz /*
3734efb46Sjohn stultz * This file contains the functions which manage clocksource drivers.
4734efb46Sjohn stultz *
5734efb46Sjohn stultz * Copyright (C) 2004, 2005 IBM, John Stultz (johnstul@us.ibm.com)
6734efb46Sjohn stultz */
7734efb46Sjohn stultz
845bbfe64SJoe Perches #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
945bbfe64SJoe Perches
10d369a5d8SKay Sievers #include <linux/device.h>
11734efb46Sjohn stultz #include <linux/clocksource.h>
12734efb46Sjohn stultz #include <linux/init.h>
13734efb46Sjohn stultz #include <linux/module.h>
14dc29a365SMathieu Desnoyers #include <linux/sched.h> /* for spin_unlock_irq() using preempt_count() m68k */
1579bf2bb3SThomas Gleixner #include <linux/tick.h>
1601548f4dSMartin Schwidefsky #include <linux/kthread.h>
17fa218f1cSPaul E. McKenney #include <linux/prandom.h>
18fa218f1cSPaul E. McKenney #include <linux/cpu.h>
19734efb46Sjohn stultz
20c1797bafSThomas Gleixner #include "tick-internal.h"
213a978377SThomas Gleixner #include "timekeeping_internal.h"
2203e13cf5SThomas Gleixner
237d2f944aSThomas Gleixner /**
247d2f944aSThomas Gleixner * clocks_calc_mult_shift - calculate mult/shift factors for scaled math of clocks
257d2f944aSThomas Gleixner * @mult: pointer to mult variable
267d2f944aSThomas Gleixner * @shift: pointer to shift variable
277d2f944aSThomas Gleixner * @from: frequency to convert from
287d2f944aSThomas Gleixner * @to: frequency to convert to
295fdade95SNicolas Pitre * @maxsec: guaranteed runtime conversion range in seconds
307d2f944aSThomas Gleixner *
317d2f944aSThomas Gleixner * The function evaluates the shift/mult pair for the scaled math
327d2f944aSThomas Gleixner * operations of clocksources and clockevents.
337d2f944aSThomas Gleixner *
347d2f944aSThomas Gleixner * @to and @from are frequency values in HZ. For clock sources @to is
357d2f944aSThomas Gleixner * NSEC_PER_SEC == 1GHz and @from is the counter frequency. For clock
367d2f944aSThomas Gleixner * event @to is the counter frequency and @from is NSEC_PER_SEC.
377d2f944aSThomas Gleixner *
385fdade95SNicolas Pitre * The @maxsec conversion range argument controls the time frame in
397d2f944aSThomas Gleixner * seconds which must be covered by the runtime conversion with the
407d2f944aSThomas Gleixner * calculated mult and shift factors. This guarantees that no 64bit
417d2f944aSThomas Gleixner * overflow happens when the input value of the conversion is
427d2f944aSThomas Gleixner * multiplied with the calculated mult factor. Larger ranges may
434bf07f65SIngo Molnar * reduce the conversion accuracy by choosing smaller mult and shift
447d2f944aSThomas Gleixner * factors.
457d2f944aSThomas Gleixner */
467d2f944aSThomas Gleixner void
clocks_calc_mult_shift(u32 * mult,u32 * shift,u32 from,u32 to,u32 maxsec)475fdade95SNicolas Pitre clocks_calc_mult_shift(u32 *mult, u32 *shift, u32 from, u32 to, u32 maxsec)
487d2f944aSThomas Gleixner {
497d2f944aSThomas Gleixner u64 tmp;
507d2f944aSThomas Gleixner u32 sft, sftacc= 32;
517d2f944aSThomas Gleixner
527d2f944aSThomas Gleixner /*
537d2f944aSThomas Gleixner * Calculate the shift factor which is limiting the conversion
547d2f944aSThomas Gleixner * range:
557d2f944aSThomas Gleixner */
565fdade95SNicolas Pitre tmp = ((u64)maxsec * from) >> 32;
577d2f944aSThomas Gleixner while (tmp) {
587d2f944aSThomas Gleixner tmp >>=1;
597d2f944aSThomas Gleixner sftacc--;
607d2f944aSThomas Gleixner }
617d2f944aSThomas Gleixner
627d2f944aSThomas Gleixner /*
637d2f944aSThomas Gleixner * Find the conversion shift/mult pair which has the best
647d2f944aSThomas Gleixner * accuracy and fits the maxsec conversion range:
657d2f944aSThomas Gleixner */
667d2f944aSThomas Gleixner for (sft = 32; sft > 0; sft--) {
677d2f944aSThomas Gleixner tmp = (u64) to << sft;
68b5776c4aSjohn stultz tmp += from / 2;
697d2f944aSThomas Gleixner do_div(tmp, from);
707d2f944aSThomas Gleixner if ((tmp >> sftacc) == 0)
717d2f944aSThomas Gleixner break;
727d2f944aSThomas Gleixner }
737d2f944aSThomas Gleixner *mult = tmp;
747d2f944aSThomas Gleixner *shift = sft;
757d2f944aSThomas Gleixner }
765304121aSMurali Karicheri EXPORT_SYMBOL_GPL(clocks_calc_mult_shift);
777d2f944aSThomas Gleixner
78734efb46Sjohn stultz /*[Clocksource internal variables]---------
79734efb46Sjohn stultz * curr_clocksource:
80f1b82746SMartin Schwidefsky * currently selected clocksource.
8139232ed5SBaolin Wang * suspend_clocksource:
8239232ed5SBaolin Wang * used to calculate the suspend time.
83734efb46Sjohn stultz * clocksource_list:
84734efb46Sjohn stultz * linked list with the registered clocksources
8575c5158fSMartin Schwidefsky * clocksource_mutex:
8675c5158fSMartin Schwidefsky * protects manipulations to curr_clocksource and the clocksource_list
87734efb46Sjohn stultz * override_name:
88734efb46Sjohn stultz * Name of the user-specified clocksource.
89734efb46Sjohn stultz */
90f1b82746SMartin Schwidefsky static struct clocksource *curr_clocksource;
9139232ed5SBaolin Wang static struct clocksource *suspend_clocksource;
92734efb46Sjohn stultz static LIST_HEAD(clocksource_list);
9375c5158fSMartin Schwidefsky static DEFINE_MUTEX(clocksource_mutex);
9429b54078SThomas Gleixner static char override_name[CS_NAME_LEN];
9554a6bc0bSThomas Gleixner static int finished_booting;
9639232ed5SBaolin Wang static u64 suspend_start;
97734efb46Sjohn stultz
982e27e793SPaul E. McKenney /*
99c37e85c1SPaul E. McKenney * Interval: 0.5sec.
100c37e85c1SPaul E. McKenney */
101c37e85c1SPaul E. McKenney #define WATCHDOG_INTERVAL (HZ >> 1)
102*af7ab5daSJiri Wiesner #define WATCHDOG_INTERVAL_MAX_NS ((2 * WATCHDOG_INTERVAL) * (NSEC_PER_SEC / HZ))
103c37e85c1SPaul E. McKenney
104c37e85c1SPaul E. McKenney /*
1052e27e793SPaul E. McKenney * Threshold: 0.0312s, when doubled: 0.0625s.
1062e27e793SPaul E. McKenney * Also a default for cs->uncertainty_margin when registering clocks.
1072e27e793SPaul E. McKenney */
1082e27e793SPaul E. McKenney #define WATCHDOG_THRESHOLD (NSEC_PER_SEC >> 5)
1092e27e793SPaul E. McKenney
1102e27e793SPaul E. McKenney /*
1112e27e793SPaul E. McKenney * Maximum permissible delay between two readouts of the watchdog
1122e27e793SPaul E. McKenney * clocksource surrounding a read of the clocksource being validated.
1132e27e793SPaul E. McKenney * This delay could be due to SMIs, NMIs, or to VCPU preemptions. Used as
1142e27e793SPaul E. McKenney * a lower bound for cs->uncertainty_margin values when registering clocks.
115c37e85c1SPaul E. McKenney *
116c37e85c1SPaul E. McKenney * The default of 500 parts per million is based on NTP's limits.
117c37e85c1SPaul E. McKenney * If a clocksource is good enough for NTP, it is good enough for us!
1182e27e793SPaul E. McKenney */
119fc153c1cSWaiman Long #ifdef CONFIG_CLOCKSOURCE_WATCHDOG_MAX_SKEW_US
120fc153c1cSWaiman Long #define MAX_SKEW_USEC CONFIG_CLOCKSOURCE_WATCHDOG_MAX_SKEW_US
121fc153c1cSWaiman Long #else
122c37e85c1SPaul E. McKenney #define MAX_SKEW_USEC (125 * WATCHDOG_INTERVAL / HZ)
123fc153c1cSWaiman Long #endif
124fc153c1cSWaiman Long
125fc153c1cSWaiman Long #define WATCHDOG_MAX_SKEW (MAX_SKEW_USEC * NSEC_PER_USEC)
1262e27e793SPaul E. McKenney
1275d8b34fdSThomas Gleixner #ifdef CONFIG_CLOCKSOURCE_WATCHDOG
128f79e0258SMartin Schwidefsky static void clocksource_watchdog_work(struct work_struct *work);
129332962f2SThomas Gleixner static void clocksource_select(void);
130f79e0258SMartin Schwidefsky
1315d8b34fdSThomas Gleixner static LIST_HEAD(watchdog_list);
1325d8b34fdSThomas Gleixner static struct clocksource *watchdog;
1335d8b34fdSThomas Gleixner static struct timer_list watchdog_timer;
134f79e0258SMartin Schwidefsky static DECLARE_WORK(watchdog_work, clocksource_watchdog_work);
1355d8b34fdSThomas Gleixner static DEFINE_SPINLOCK(watchdog_lock);
136fb63a0ebSMartin Schwidefsky static int watchdog_running;
1379fb60336SThomas Gleixner static atomic_t watchdog_reset_pending;
138*af7ab5daSJiri Wiesner static int64_t watchdog_max_interval;
139b52f52a0SThomas Gleixner
clocksource_watchdog_lock(unsigned long * flags)1400f48b41fSMathieu Malaterre static inline void clocksource_watchdog_lock(unsigned long *flags)
1412aae7bcfSPeter Zijlstra {
1422aae7bcfSPeter Zijlstra spin_lock_irqsave(&watchdog_lock, *flags);
1432aae7bcfSPeter Zijlstra }
1442aae7bcfSPeter Zijlstra
clocksource_watchdog_unlock(unsigned long * flags)1450f48b41fSMathieu Malaterre static inline void clocksource_watchdog_unlock(unsigned long *flags)
1462aae7bcfSPeter Zijlstra {
1472aae7bcfSPeter Zijlstra spin_unlock_irqrestore(&watchdog_lock, *flags);
1482aae7bcfSPeter Zijlstra }
1492aae7bcfSPeter Zijlstra
150e2c631baSPeter Zijlstra static int clocksource_watchdog_kthread(void *data);
151e2c631baSPeter Zijlstra static void __clocksource_change_rating(struct clocksource *cs, int rating);
152e2c631baSPeter Zijlstra
clocksource_watchdog_work(struct work_struct * work)153e2c631baSPeter Zijlstra static void clocksource_watchdog_work(struct work_struct *work)
154e2c631baSPeter Zijlstra {
155e2c631baSPeter Zijlstra /*
156e2c631baSPeter Zijlstra * We cannot directly run clocksource_watchdog_kthread() here, because
157e2c631baSPeter Zijlstra * clocksource_select() calls timekeeping_notify() which uses
158e2c631baSPeter Zijlstra * stop_machine(). One cannot use stop_machine() from a workqueue() due
159e2c631baSPeter Zijlstra * lock inversions wrt CPU hotplug.
160e2c631baSPeter Zijlstra *
161e2c631baSPeter Zijlstra * Also, we only ever run this work once or twice during the lifetime
162e2c631baSPeter Zijlstra * of the kernel, so there is no point in creating a more permanent
163e2c631baSPeter Zijlstra * kthread for this.
164e2c631baSPeter Zijlstra *
165e2c631baSPeter Zijlstra * If kthread_run fails the next watchdog scan over the
166e2c631baSPeter Zijlstra * watchdog_list will find the unstable clock again.
167e2c631baSPeter Zijlstra */
168e2c631baSPeter Zijlstra kthread_run(clocksource_watchdog_kthread, NULL, "kwatchdog");
169e2c631baSPeter Zijlstra }
170e2c631baSPeter Zijlstra
__clocksource_unstable(struct clocksource * cs)1717285dd7fSThomas Gleixner static void __clocksource_unstable(struct clocksource *cs)
1727285dd7fSThomas Gleixner {
1737285dd7fSThomas Gleixner cs->flags &= ~(CLOCK_SOURCE_VALID_FOR_HRES | CLOCK_SOURCE_WATCHDOG);
1747285dd7fSThomas Gleixner cs->flags |= CLOCK_SOURCE_UNSTABLE;
17512907fbbSThomas Gleixner
176cd2af07dSPeter Zijlstra /*
177e2c631baSPeter Zijlstra * If the clocksource is registered clocksource_watchdog_kthread() will
178cd2af07dSPeter Zijlstra * re-rate and re-select.
179cd2af07dSPeter Zijlstra */
180cd2af07dSPeter Zijlstra if (list_empty(&cs->list)) {
181cd2af07dSPeter Zijlstra cs->rating = 0;
1822aae7bcfSPeter Zijlstra return;
183cd2af07dSPeter Zijlstra }
1842aae7bcfSPeter Zijlstra
18512907fbbSThomas Gleixner if (cs->mark_unstable)
18612907fbbSThomas Gleixner cs->mark_unstable(cs);
18712907fbbSThomas Gleixner
188e2c631baSPeter Zijlstra /* kick clocksource_watchdog_kthread() */
18954a6bc0bSThomas Gleixner if (finished_booting)
1907285dd7fSThomas Gleixner schedule_work(&watchdog_work);
1917285dd7fSThomas Gleixner }
1927285dd7fSThomas Gleixner
1937285dd7fSThomas Gleixner /**
1947285dd7fSThomas Gleixner * clocksource_mark_unstable - mark clocksource unstable via watchdog
1957285dd7fSThomas Gleixner * @cs: clocksource to be marked unstable
1967285dd7fSThomas Gleixner *
1977dba33c6SPeter Zijlstra * This function is called by the x86 TSC code to mark clocksources as unstable;
198e2c631baSPeter Zijlstra * it defers demotion and re-selection to a kthread.
1997285dd7fSThomas Gleixner */
clocksource_mark_unstable(struct clocksource * cs)2007285dd7fSThomas Gleixner void clocksource_mark_unstable(struct clocksource *cs)
2017285dd7fSThomas Gleixner {
2027285dd7fSThomas Gleixner unsigned long flags;
2037285dd7fSThomas Gleixner
2047285dd7fSThomas Gleixner spin_lock_irqsave(&watchdog_lock, flags);
2057285dd7fSThomas Gleixner if (!(cs->flags & CLOCK_SOURCE_UNSTABLE)) {
2062aae7bcfSPeter Zijlstra if (!list_empty(&cs->list) && list_empty(&cs->wd_list))
2077285dd7fSThomas Gleixner list_add(&cs->wd_list, &watchdog_list);
2087285dd7fSThomas Gleixner __clocksource_unstable(cs);
2097285dd7fSThomas Gleixner }
2107285dd7fSThomas Gleixner spin_unlock_irqrestore(&watchdog_lock, flags);
2115d8b34fdSThomas Gleixner }
2125d8b34fdSThomas Gleixner
2131a562067SWaiman Long ulong max_cswd_read_retries = 2;
214db3a34e1SPaul E. McKenney module_param(max_cswd_read_retries, ulong, 0644);
2151253b9b8SPaul E. McKenney EXPORT_SYMBOL_GPL(max_cswd_read_retries);
216fa218f1cSPaul E. McKenney static int verify_n_cpus = 8;
217fa218f1cSPaul E. McKenney module_param(verify_n_cpus, int, 0644);
218db3a34e1SPaul E. McKenney
219c86ff8c5SWaiman Long enum wd_read_status {
220c86ff8c5SWaiman Long WD_READ_SUCCESS,
221c86ff8c5SWaiman Long WD_READ_UNSTABLE,
222c86ff8c5SWaiman Long WD_READ_SKIP
223c86ff8c5SWaiman Long };
224c86ff8c5SWaiman Long
cs_watchdog_read(struct clocksource * cs,u64 * csnow,u64 * wdnow)225c86ff8c5SWaiman Long static enum wd_read_status cs_watchdog_read(struct clocksource *cs, u64 *csnow, u64 *wdnow)
226db3a34e1SPaul E. McKenney {
227db3a34e1SPaul E. McKenney unsigned int nretries;
228c86ff8c5SWaiman Long u64 wd_end, wd_end2, wd_delta;
229c86ff8c5SWaiman Long int64_t wd_delay, wd_seq_delay;
230db3a34e1SPaul E. McKenney
231db3a34e1SPaul E. McKenney for (nretries = 0; nretries <= max_cswd_read_retries; nretries++) {
232db3a34e1SPaul E. McKenney local_irq_disable();
233db3a34e1SPaul E. McKenney *wdnow = watchdog->read(watchdog);
234db3a34e1SPaul E. McKenney *csnow = cs->read(cs);
235db3a34e1SPaul E. McKenney wd_end = watchdog->read(watchdog);
236c86ff8c5SWaiman Long wd_end2 = watchdog->read(watchdog);
237db3a34e1SPaul E. McKenney local_irq_enable();
238db3a34e1SPaul E. McKenney
239db3a34e1SPaul E. McKenney wd_delta = clocksource_delta(wd_end, *wdnow, watchdog->mask);
240db3a34e1SPaul E. McKenney wd_delay = clocksource_cyc2ns(wd_delta, watchdog->mult,
241db3a34e1SPaul E. McKenney watchdog->shift);
242db3a34e1SPaul E. McKenney if (wd_delay <= WATCHDOG_MAX_SKEW) {
243db3a34e1SPaul E. McKenney if (nretries > 1 || nretries >= max_cswd_read_retries) {
244db3a34e1SPaul E. McKenney pr_warn("timekeeping watchdog on CPU%d: %s retried %d times before success\n",
245db3a34e1SPaul E. McKenney smp_processor_id(), watchdog->name, nretries);
246db3a34e1SPaul E. McKenney }
247c86ff8c5SWaiman Long return WD_READ_SUCCESS;
248db3a34e1SPaul E. McKenney }
249c86ff8c5SWaiman Long
250c86ff8c5SWaiman Long /*
251c86ff8c5SWaiman Long * Now compute delay in consecutive watchdog read to see if
252c86ff8c5SWaiman Long * there is too much external interferences that cause
253c86ff8c5SWaiman Long * significant delay in reading both clocksource and watchdog.
254c86ff8c5SWaiman Long *
255c86ff8c5SWaiman Long * If consecutive WD read-back delay > WATCHDOG_MAX_SKEW/2,
256c86ff8c5SWaiman Long * report system busy, reinit the watchdog and skip the current
257c86ff8c5SWaiman Long * watchdog test.
258c86ff8c5SWaiman Long */
259c86ff8c5SWaiman Long wd_delta = clocksource_delta(wd_end2, wd_end, watchdog->mask);
260c86ff8c5SWaiman Long wd_seq_delay = clocksource_cyc2ns(wd_delta, watchdog->mult, watchdog->shift);
261c86ff8c5SWaiman Long if (wd_seq_delay > WATCHDOG_MAX_SKEW/2)
262c86ff8c5SWaiman Long goto skip_test;
263db3a34e1SPaul E. McKenney }
264db3a34e1SPaul E. McKenney
265f092eb34SPaul E. McKenney pr_warn("timekeeping watchdog on CPU%d: wd-%s-wd excessive read-back delay of %lldns vs. limit of %ldns, wd-wd read-back delay only %lldns, attempt %d, marking %s unstable\n",
266f092eb34SPaul E. McKenney smp_processor_id(), cs->name, wd_delay, WATCHDOG_MAX_SKEW, wd_seq_delay, nretries, cs->name);
267c86ff8c5SWaiman Long return WD_READ_UNSTABLE;
268c86ff8c5SWaiman Long
269c86ff8c5SWaiman Long skip_test:
270c86ff8c5SWaiman Long pr_info("timekeeping watchdog on CPU%d: %s wd-wd read-back delay of %lldns\n",
271c86ff8c5SWaiman Long smp_processor_id(), watchdog->name, wd_seq_delay);
272c86ff8c5SWaiman Long pr_info("wd-%s-wd read-back delay of %lldns, clock-skew test skipped!\n",
273c86ff8c5SWaiman Long cs->name, wd_delay);
274c86ff8c5SWaiman Long return WD_READ_SKIP;
275db3a34e1SPaul E. McKenney }
276db3a34e1SPaul E. McKenney
2777560c02bSPaul E. McKenney static u64 csnow_mid;
2787560c02bSPaul E. McKenney static cpumask_t cpus_ahead;
2797560c02bSPaul E. McKenney static cpumask_t cpus_behind;
280fa218f1cSPaul E. McKenney static cpumask_t cpus_chosen;
281fa218f1cSPaul E. McKenney
clocksource_verify_choose_cpus(void)282fa218f1cSPaul E. McKenney static void clocksource_verify_choose_cpus(void)
283fa218f1cSPaul E. McKenney {
284fa218f1cSPaul E. McKenney int cpu, i, n = verify_n_cpus;
285fa218f1cSPaul E. McKenney
286fa218f1cSPaul E. McKenney if (n < 0) {
287fa218f1cSPaul E. McKenney /* Check all of the CPUs. */
288fa218f1cSPaul E. McKenney cpumask_copy(&cpus_chosen, cpu_online_mask);
289fa218f1cSPaul E. McKenney cpumask_clear_cpu(smp_processor_id(), &cpus_chosen);
290fa218f1cSPaul E. McKenney return;
291fa218f1cSPaul E. McKenney }
292fa218f1cSPaul E. McKenney
293fa218f1cSPaul E. McKenney /* If no checking desired, or no other CPU to check, leave. */
294fa218f1cSPaul E. McKenney cpumask_clear(&cpus_chosen);
295fa218f1cSPaul E. McKenney if (n == 0 || num_online_cpus() <= 1)
296fa218f1cSPaul E. McKenney return;
297fa218f1cSPaul E. McKenney
298fa218f1cSPaul E. McKenney /* Make sure to select at least one CPU other than the current CPU. */
2999b51d9d8SYury Norov cpu = cpumask_first(cpu_online_mask);
300fa218f1cSPaul E. McKenney if (cpu == smp_processor_id())
301fa218f1cSPaul E. McKenney cpu = cpumask_next(cpu, cpu_online_mask);
302fa218f1cSPaul E. McKenney if (WARN_ON_ONCE(cpu >= nr_cpu_ids))
303fa218f1cSPaul E. McKenney return;
304fa218f1cSPaul E. McKenney cpumask_set_cpu(cpu, &cpus_chosen);
305fa218f1cSPaul E. McKenney
306fa218f1cSPaul E. McKenney /* Force a sane value for the boot parameter. */
307fa218f1cSPaul E. McKenney if (n > nr_cpu_ids)
308fa218f1cSPaul E. McKenney n = nr_cpu_ids;
309fa218f1cSPaul E. McKenney
310fa218f1cSPaul E. McKenney /*
311fa218f1cSPaul E. McKenney * Randomly select the specified number of CPUs. If the same
312fa218f1cSPaul E. McKenney * CPU is selected multiple times, that CPU is checked only once,
313fa218f1cSPaul E. McKenney * and no replacement CPU is selected. This gracefully handles
314fa218f1cSPaul E. McKenney * situations where verify_n_cpus is greater than the number of
315fa218f1cSPaul E. McKenney * CPUs that are currently online.
316fa218f1cSPaul E. McKenney */
317fa218f1cSPaul E. McKenney for (i = 1; i < n; i++) {
3188032bf12SJason A. Donenfeld cpu = get_random_u32_below(nr_cpu_ids);
319fa218f1cSPaul E. McKenney cpu = cpumask_next(cpu - 1, cpu_online_mask);
320fa218f1cSPaul E. McKenney if (cpu >= nr_cpu_ids)
3219b51d9d8SYury Norov cpu = cpumask_first(cpu_online_mask);
322fa218f1cSPaul E. McKenney if (!WARN_ON_ONCE(cpu >= nr_cpu_ids))
323fa218f1cSPaul E. McKenney cpumask_set_cpu(cpu, &cpus_chosen);
324fa218f1cSPaul E. McKenney }
325fa218f1cSPaul E. McKenney
326fa218f1cSPaul E. McKenney /* Don't verify ourselves. */
327fa218f1cSPaul E. McKenney cpumask_clear_cpu(smp_processor_id(), &cpus_chosen);
328fa218f1cSPaul E. McKenney }
3297560c02bSPaul E. McKenney
clocksource_verify_one_cpu(void * csin)3307560c02bSPaul E. McKenney static void clocksource_verify_one_cpu(void *csin)
3317560c02bSPaul E. McKenney {
3327560c02bSPaul E. McKenney struct clocksource *cs = (struct clocksource *)csin;
3337560c02bSPaul E. McKenney
3347560c02bSPaul E. McKenney csnow_mid = cs->read(cs);
3357560c02bSPaul E. McKenney }
3367560c02bSPaul E. McKenney
clocksource_verify_percpu(struct clocksource * cs)3371253b9b8SPaul E. McKenney void clocksource_verify_percpu(struct clocksource *cs)
3387560c02bSPaul E. McKenney {
3397560c02bSPaul E. McKenney int64_t cs_nsec, cs_nsec_max = 0, cs_nsec_min = LLONG_MAX;
3407560c02bSPaul E. McKenney u64 csnow_begin, csnow_end;
3417560c02bSPaul E. McKenney int cpu, testcpu;
3427560c02bSPaul E. McKenney s64 delta;
3437560c02bSPaul E. McKenney
344fa218f1cSPaul E. McKenney if (verify_n_cpus == 0)
345fa218f1cSPaul E. McKenney return;
3467560c02bSPaul E. McKenney cpumask_clear(&cpus_ahead);
3477560c02bSPaul E. McKenney cpumask_clear(&cpus_behind);
348698429f9SSebastian Andrzej Siewior cpus_read_lock();
3497560c02bSPaul E. McKenney preempt_disable();
350fa218f1cSPaul E. McKenney clocksource_verify_choose_cpus();
3518afbcaf8SYury Norov if (cpumask_empty(&cpus_chosen)) {
352fa218f1cSPaul E. McKenney preempt_enable();
353698429f9SSebastian Andrzej Siewior cpus_read_unlock();
354fa218f1cSPaul E. McKenney pr_warn("Not enough CPUs to check clocksource '%s'.\n", cs->name);
355fa218f1cSPaul E. McKenney return;
356fa218f1cSPaul E. McKenney }
3577560c02bSPaul E. McKenney testcpu = smp_processor_id();
358fa218f1cSPaul E. McKenney pr_warn("Checking clocksource %s synchronization from CPU %d to CPUs %*pbl.\n", cs->name, testcpu, cpumask_pr_args(&cpus_chosen));
359fa218f1cSPaul E. McKenney for_each_cpu(cpu, &cpus_chosen) {
3607560c02bSPaul E. McKenney if (cpu == testcpu)
3617560c02bSPaul E. McKenney continue;
3627560c02bSPaul E. McKenney csnow_begin = cs->read(cs);
3637560c02bSPaul E. McKenney smp_call_function_single(cpu, clocksource_verify_one_cpu, cs, 1);
3647560c02bSPaul E. McKenney csnow_end = cs->read(cs);
3657560c02bSPaul E. McKenney delta = (s64)((csnow_mid - csnow_begin) & cs->mask);
3667560c02bSPaul E. McKenney if (delta < 0)
3677560c02bSPaul E. McKenney cpumask_set_cpu(cpu, &cpus_behind);
3687560c02bSPaul E. McKenney delta = (csnow_end - csnow_mid) & cs->mask;
3697560c02bSPaul E. McKenney if (delta < 0)
3707560c02bSPaul E. McKenney cpumask_set_cpu(cpu, &cpus_ahead);
3717560c02bSPaul E. McKenney delta = clocksource_delta(csnow_end, csnow_begin, cs->mask);
3727560c02bSPaul E. McKenney cs_nsec = clocksource_cyc2ns(delta, cs->mult, cs->shift);
3737560c02bSPaul E. McKenney if (cs_nsec > cs_nsec_max)
3747560c02bSPaul E. McKenney cs_nsec_max = cs_nsec;
3757560c02bSPaul E. McKenney if (cs_nsec < cs_nsec_min)
3767560c02bSPaul E. McKenney cs_nsec_min = cs_nsec;
3777560c02bSPaul E. McKenney }
3787560c02bSPaul E. McKenney preempt_enable();
379698429f9SSebastian Andrzej Siewior cpus_read_unlock();
3807560c02bSPaul E. McKenney if (!cpumask_empty(&cpus_ahead))
3817560c02bSPaul E. McKenney pr_warn(" CPUs %*pbl ahead of CPU %d for clocksource %s.\n",
3827560c02bSPaul E. McKenney cpumask_pr_args(&cpus_ahead), testcpu, cs->name);
3837560c02bSPaul E. McKenney if (!cpumask_empty(&cpus_behind))
3847560c02bSPaul E. McKenney pr_warn(" CPUs %*pbl behind CPU %d for clocksource %s.\n",
3857560c02bSPaul E. McKenney cpumask_pr_args(&cpus_behind), testcpu, cs->name);
3867560c02bSPaul E. McKenney if (!cpumask_empty(&cpus_ahead) || !cpumask_empty(&cpus_behind))
3877560c02bSPaul E. McKenney pr_warn(" CPU %d check durations %lldns - %lldns for clocksource %s.\n",
3887560c02bSPaul E. McKenney testcpu, cs_nsec_min, cs_nsec_max, cs->name);
3897560c02bSPaul E. McKenney }
3901253b9b8SPaul E. McKenney EXPORT_SYMBOL_GPL(clocksource_verify_percpu);
3917560c02bSPaul E. McKenney
clocksource_reset_watchdog(void)392b7082cdfSFeng Tang static inline void clocksource_reset_watchdog(void)
393b7082cdfSFeng Tang {
394b7082cdfSFeng Tang struct clocksource *cs;
395b7082cdfSFeng Tang
396b7082cdfSFeng Tang list_for_each_entry(cs, &watchdog_list, wd_list)
397b7082cdfSFeng Tang cs->flags &= ~CLOCK_SOURCE_WATCHDOG;
398b7082cdfSFeng Tang }
399b7082cdfSFeng Tang
400b7082cdfSFeng Tang
clocksource_watchdog(struct timer_list * unused)401e99e88a9SKees Cook static void clocksource_watchdog(struct timer_list *unused)
4025d8b34fdSThomas Gleixner {
403a5a1d1c2SThomas Gleixner u64 csnow, wdnow, cslast, wdlast, delta;
404*af7ab5daSJiri Wiesner int64_t wd_nsec, cs_nsec, interval;
4059fb60336SThomas Gleixner int next_cpu, reset_pending;
406db3a34e1SPaul E. McKenney struct clocksource *cs;
407c86ff8c5SWaiman Long enum wd_read_status read_ret;
408b7082cdfSFeng Tang unsigned long extra_wait = 0;
4092e27e793SPaul E. McKenney u32 md;
4105d8b34fdSThomas Gleixner
4115d8b34fdSThomas Gleixner spin_lock(&watchdog_lock);
412fb63a0ebSMartin Schwidefsky if (!watchdog_running)
413fb63a0ebSMartin Schwidefsky goto out;
4145d8b34fdSThomas Gleixner
4159fb60336SThomas Gleixner reset_pending = atomic_read(&watchdog_reset_pending);
4169fb60336SThomas Gleixner
417c55c87c8SMartin Schwidefsky list_for_each_entry(cs, &watchdog_list, wd_list) {
418c55c87c8SMartin Schwidefsky
419c55c87c8SMartin Schwidefsky /* Clocksource already marked unstable? */
42001548f4dSMartin Schwidefsky if (cs->flags & CLOCK_SOURCE_UNSTABLE) {
42154a6bc0bSThomas Gleixner if (finished_booting)
42201548f4dSMartin Schwidefsky schedule_work(&watchdog_work);
423c55c87c8SMartin Schwidefsky continue;
42401548f4dSMartin Schwidefsky }
425c55c87c8SMartin Schwidefsky
426c86ff8c5SWaiman Long read_ret = cs_watchdog_read(cs, &csnow, &wdnow);
427c86ff8c5SWaiman Long
428b7082cdfSFeng Tang if (read_ret == WD_READ_UNSTABLE) {
429db3a34e1SPaul E. McKenney /* Clock readout unreliable, so give it up. */
430db3a34e1SPaul E. McKenney __clocksource_unstable(cs);
431db3a34e1SPaul E. McKenney continue;
432db3a34e1SPaul E. McKenney }
433b52f52a0SThomas Gleixner
434b7082cdfSFeng Tang /*
435b7082cdfSFeng Tang * When WD_READ_SKIP is returned, it means the system is likely
436b7082cdfSFeng Tang * under very heavy load, where the latency of reading
437b7082cdfSFeng Tang * watchdog/clocksource is very big, and affect the accuracy of
438b7082cdfSFeng Tang * watchdog check. So give system some space and suspend the
439b7082cdfSFeng Tang * watchdog check for 5 minutes.
440b7082cdfSFeng Tang */
441b7082cdfSFeng Tang if (read_ret == WD_READ_SKIP) {
442b7082cdfSFeng Tang /*
443b7082cdfSFeng Tang * As the watchdog timer will be suspended, and
444b7082cdfSFeng Tang * cs->last could keep unchanged for 5 minutes, reset
445b7082cdfSFeng Tang * the counters.
446b7082cdfSFeng Tang */
447b7082cdfSFeng Tang clocksource_reset_watchdog();
448b7082cdfSFeng Tang extra_wait = HZ * 300;
449b7082cdfSFeng Tang break;
450b7082cdfSFeng Tang }
451b7082cdfSFeng Tang
4528cf4e750SMartin Schwidefsky /* Clocksource initialized ? */
4539fb60336SThomas Gleixner if (!(cs->flags & CLOCK_SOURCE_WATCHDOG) ||
4549fb60336SThomas Gleixner atomic_read(&watchdog_reset_pending)) {
4558cf4e750SMartin Schwidefsky cs->flags |= CLOCK_SOURCE_WATCHDOG;
456b5199515SThomas Gleixner cs->wd_last = wdnow;
457b5199515SThomas Gleixner cs->cs_last = csnow;
458b52f52a0SThomas Gleixner continue;
459b52f52a0SThomas Gleixner }
460b52f52a0SThomas Gleixner
4613a978377SThomas Gleixner delta = clocksource_delta(wdnow, cs->wd_last, watchdog->mask);
4623a978377SThomas Gleixner wd_nsec = clocksource_cyc2ns(delta, watchdog->mult,
4633a978377SThomas Gleixner watchdog->shift);
464b5199515SThomas Gleixner
4653a978377SThomas Gleixner delta = clocksource_delta(csnow, cs->cs_last, cs->mask);
4663a978377SThomas Gleixner cs_nsec = clocksource_cyc2ns(delta, cs->mult, cs->shift);
4670b046b21SJohn Stultz wdlast = cs->wd_last; /* save these in case we print them */
4680b046b21SJohn Stultz cslast = cs->cs_last;
469b5199515SThomas Gleixner cs->cs_last = csnow;
470b5199515SThomas Gleixner cs->wd_last = wdnow;
471b5199515SThomas Gleixner
4729fb60336SThomas Gleixner if (atomic_read(&watchdog_reset_pending))
4739fb60336SThomas Gleixner continue;
4749fb60336SThomas Gleixner
475*af7ab5daSJiri Wiesner /*
476*af7ab5daSJiri Wiesner * The processing of timer softirqs can get delayed (usually
477*af7ab5daSJiri Wiesner * on account of ksoftirqd not getting to run in a timely
478*af7ab5daSJiri Wiesner * manner), which causes the watchdog interval to stretch.
479*af7ab5daSJiri Wiesner * Skew detection may fail for longer watchdog intervals
480*af7ab5daSJiri Wiesner * on account of fixed margins being used.
481*af7ab5daSJiri Wiesner * Some clocksources, e.g. acpi_pm, cannot tolerate
482*af7ab5daSJiri Wiesner * watchdog intervals longer than a few seconds.
483*af7ab5daSJiri Wiesner */
484*af7ab5daSJiri Wiesner interval = max(cs_nsec, wd_nsec);
485*af7ab5daSJiri Wiesner if (unlikely(interval > WATCHDOG_INTERVAL_MAX_NS)) {
486*af7ab5daSJiri Wiesner if (system_state > SYSTEM_SCHEDULING &&
487*af7ab5daSJiri Wiesner interval > 2 * watchdog_max_interval) {
488*af7ab5daSJiri Wiesner watchdog_max_interval = interval;
489*af7ab5daSJiri Wiesner pr_warn("Long readout interval, skipping watchdog check: cs_nsec: %lld wd_nsec: %lld\n",
490*af7ab5daSJiri Wiesner cs_nsec, wd_nsec);
491*af7ab5daSJiri Wiesner }
492*af7ab5daSJiri Wiesner watchdog_timer.expires = jiffies;
493*af7ab5daSJiri Wiesner continue;
494*af7ab5daSJiri Wiesner }
495*af7ab5daSJiri Wiesner
496b5199515SThomas Gleixner /* Check the deviation from the watchdog clocksource. */
4972e27e793SPaul E. McKenney md = cs->uncertainty_margin + watchdog->uncertainty_margin;
4982e27e793SPaul E. McKenney if (abs(cs_nsec - wd_nsec) > md) {
499e40806e9SPaul E. McKenney s64 cs_wd_msec;
500e40806e9SPaul E. McKenney s64 wd_msec;
501dd029269SPaul E. McKenney u32 wd_rem;
502dd029269SPaul E. McKenney
503390dd67cSSeiichi Ikarashi pr_warn("timekeeping watchdog on CPU%d: Marking clocksource '%s' as unstable because the skew is too large:\n",
504390dd67cSSeiichi Ikarashi smp_processor_id(), cs->name);
50522a22383SFeng Tang pr_warn(" '%s' wd_nsec: %lld wd_now: %llx wd_last: %llx mask: %llx\n",
50622a22383SFeng Tang watchdog->name, wd_nsec, wdnow, wdlast, watchdog->mask);
50722a22383SFeng Tang pr_warn(" '%s' cs_nsec: %lld cs_now: %llx cs_last: %llx mask: %llx\n",
50822a22383SFeng Tang cs->name, cs_nsec, csnow, cslast, cs->mask);
509e40806e9SPaul E. McKenney cs_wd_msec = div_s64_rem(cs_nsec - wd_nsec, 1000 * 1000, &wd_rem);
510e40806e9SPaul E. McKenney wd_msec = div_s64_rem(wd_nsec, 1000 * 1000, &wd_rem);
511dd029269SPaul E. McKenney pr_warn(" Clocksource '%s' skewed %lld ns (%lld ms) over watchdog '%s' interval of %lld ns (%lld ms)\n",
512dd029269SPaul E. McKenney cs->name, cs_nsec - wd_nsec, cs_wd_msec, watchdog->name, wd_nsec, wd_msec);
513fa218f1cSPaul E. McKenney if (curr_clocksource == cs)
514fa218f1cSPaul E. McKenney pr_warn(" '%s' is current clocksource.\n", cs->name);
515fa218f1cSPaul E. McKenney else if (curr_clocksource)
516fa218f1cSPaul E. McKenney pr_warn(" '%s' (not '%s') is current clocksource.\n", curr_clocksource->name, cs->name);
517fa218f1cSPaul E. McKenney else
518fa218f1cSPaul E. McKenney pr_warn(" No current clocksource.\n");
5190b046b21SJohn Stultz __clocksource_unstable(cs);
5208cf4e750SMartin Schwidefsky continue;
5218cf4e750SMartin Schwidefsky }
5228cf4e750SMartin Schwidefsky
523b421b22bSPeter Zijlstra if (cs == curr_clocksource && cs->tick_stable)
524b421b22bSPeter Zijlstra cs->tick_stable(cs);
525b421b22bSPeter Zijlstra
5268cf4e750SMartin Schwidefsky if (!(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) &&
5278cf4e750SMartin Schwidefsky (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) &&
5285d8b34fdSThomas Gleixner (watchdog->flags & CLOCK_SOURCE_IS_CONTINUOUS)) {
529332962f2SThomas Gleixner /* Mark it valid for high-res. */
5305d8b34fdSThomas Gleixner cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;
531332962f2SThomas Gleixner
53279bf2bb3SThomas Gleixner /*
533332962f2SThomas Gleixner * clocksource_done_booting() will sort it if
534332962f2SThomas Gleixner * finished_booting is not set yet.
53579bf2bb3SThomas Gleixner */
536332962f2SThomas Gleixner if (!finished_booting)
537332962f2SThomas Gleixner continue;
538332962f2SThomas Gleixner
539332962f2SThomas Gleixner /*
540332962f2SThomas Gleixner * If this is not the current clocksource let
541332962f2SThomas Gleixner * the watchdog thread reselect it. Due to the
542332962f2SThomas Gleixner * change to high res this clocksource might
543332962f2SThomas Gleixner * be preferred now. If it is the current
544332962f2SThomas Gleixner * clocksource let the tick code know about
545332962f2SThomas Gleixner * that change.
546332962f2SThomas Gleixner */
547332962f2SThomas Gleixner if (cs != curr_clocksource) {
548332962f2SThomas Gleixner cs->flags |= CLOCK_SOURCE_RESELECT;
549332962f2SThomas Gleixner schedule_work(&watchdog_work);
550332962f2SThomas Gleixner } else {
55179bf2bb3SThomas Gleixner tick_clock_notify();
5525d8b34fdSThomas Gleixner }
5535d8b34fdSThomas Gleixner }
554332962f2SThomas Gleixner }
5555d8b34fdSThomas Gleixner
5566993fc5bSAndi Kleen /*
5579fb60336SThomas Gleixner * We only clear the watchdog_reset_pending, when we did a
5589fb60336SThomas Gleixner * full cycle through all clocksources.
5599fb60336SThomas Gleixner */
5609fb60336SThomas Gleixner if (reset_pending)
5619fb60336SThomas Gleixner atomic_dec(&watchdog_reset_pending);
5629fb60336SThomas Gleixner
5639fb60336SThomas Gleixner /*
564c55c87c8SMartin Schwidefsky * Cycle through CPUs to check if the CPUs stay synchronized
565c55c87c8SMartin Schwidefsky * to each other.
5666993fc5bSAndi Kleen */
567c55c87c8SMartin Schwidefsky next_cpu = cpumask_next(raw_smp_processor_id(), cpu_online_mask);
568cad0e458SMike Travis if (next_cpu >= nr_cpu_ids)
5696b954823SRusty Russell next_cpu = cpumask_first(cpu_online_mask);
570febac332SKonstantin Khlebnikov
571febac332SKonstantin Khlebnikov /*
572febac332SKonstantin Khlebnikov * Arm timer if not already pending: could race with concurrent
573febac332SKonstantin Khlebnikov * pair clocksource_stop_watchdog() clocksource_start_watchdog().
574febac332SKonstantin Khlebnikov */
575febac332SKonstantin Khlebnikov if (!timer_pending(&watchdog_timer)) {
576b7082cdfSFeng Tang watchdog_timer.expires += WATCHDOG_INTERVAL + extra_wait;
5776993fc5bSAndi Kleen add_timer_on(&watchdog_timer, next_cpu);
578febac332SKonstantin Khlebnikov }
579fb63a0ebSMartin Schwidefsky out:
5805d8b34fdSThomas Gleixner spin_unlock(&watchdog_lock);
5815d8b34fdSThomas Gleixner }
5820f8e8ef7SMartin Schwidefsky
clocksource_start_watchdog(void)583fb63a0ebSMartin Schwidefsky static inline void clocksource_start_watchdog(void)
584fb63a0ebSMartin Schwidefsky {
585fb63a0ebSMartin Schwidefsky if (watchdog_running || !watchdog || list_empty(&watchdog_list))
586fb63a0ebSMartin Schwidefsky return;
587e99e88a9SKees Cook timer_setup(&watchdog_timer, clocksource_watchdog, 0);
588fb63a0ebSMartin Schwidefsky watchdog_timer.expires = jiffies + WATCHDOG_INTERVAL;
589fb63a0ebSMartin Schwidefsky add_timer_on(&watchdog_timer, cpumask_first(cpu_online_mask));
590fb63a0ebSMartin Schwidefsky watchdog_running = 1;
591fb63a0ebSMartin Schwidefsky }
592fb63a0ebSMartin Schwidefsky
clocksource_stop_watchdog(void)593fb63a0ebSMartin Schwidefsky static inline void clocksource_stop_watchdog(void)
594fb63a0ebSMartin Schwidefsky {
595fb63a0ebSMartin Schwidefsky if (!watchdog_running || (watchdog && !list_empty(&watchdog_list)))
596fb63a0ebSMartin Schwidefsky return;
597fb63a0ebSMartin Schwidefsky del_timer(&watchdog_timer);
598fb63a0ebSMartin Schwidefsky watchdog_running = 0;
599fb63a0ebSMartin Schwidefsky }
600fb63a0ebSMartin Schwidefsky
clocksource_resume_watchdog(void)601b52f52a0SThomas Gleixner static void clocksource_resume_watchdog(void)
602b52f52a0SThomas Gleixner {
6039fb60336SThomas Gleixner atomic_inc(&watchdog_reset_pending);
604b52f52a0SThomas Gleixner }
605b52f52a0SThomas Gleixner
clocksource_enqueue_watchdog(struct clocksource * cs)606fb63a0ebSMartin Schwidefsky static void clocksource_enqueue_watchdog(struct clocksource *cs)
6075d8b34fdSThomas Gleixner {
6085b9e886aSPeter Zijlstra INIT_LIST_HEAD(&cs->wd_list);
6095b9e886aSPeter Zijlstra
6105d8b34fdSThomas Gleixner if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) {
611fb63a0ebSMartin Schwidefsky /* cs is a clocksource to be watched. */
6125d8b34fdSThomas Gleixner list_add(&cs->wd_list, &watchdog_list);
613fb63a0ebSMartin Schwidefsky cs->flags &= ~CLOCK_SOURCE_WATCHDOG;
614948ac6d7SThomas Gleixner } else {
615fb63a0ebSMartin Schwidefsky /* cs is a watchdog. */
616948ac6d7SThomas Gleixner if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS)
6175d8b34fdSThomas Gleixner cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;
618bbf66d89SVitaly Kuznetsov }
619bbf66d89SVitaly Kuznetsov }
620bbf66d89SVitaly Kuznetsov
clocksource_select_watchdog(bool fallback)621bbf66d89SVitaly Kuznetsov static void clocksource_select_watchdog(bool fallback)
622bbf66d89SVitaly Kuznetsov {
623bbf66d89SVitaly Kuznetsov struct clocksource *cs, *old_wd;
624bbf66d89SVitaly Kuznetsov unsigned long flags;
625bbf66d89SVitaly Kuznetsov
626bbf66d89SVitaly Kuznetsov spin_lock_irqsave(&watchdog_lock, flags);
627bbf66d89SVitaly Kuznetsov /* save current watchdog */
628bbf66d89SVitaly Kuznetsov old_wd = watchdog;
629bbf66d89SVitaly Kuznetsov if (fallback)
630bbf66d89SVitaly Kuznetsov watchdog = NULL;
631bbf66d89SVitaly Kuznetsov
632bbf66d89SVitaly Kuznetsov list_for_each_entry(cs, &clocksource_list, list) {
633bbf66d89SVitaly Kuznetsov /* cs is a clocksource to be watched. */
634bbf66d89SVitaly Kuznetsov if (cs->flags & CLOCK_SOURCE_MUST_VERIFY)
635bbf66d89SVitaly Kuznetsov continue;
636bbf66d89SVitaly Kuznetsov
637bbf66d89SVitaly Kuznetsov /* Skip current if we were requested for a fallback. */
638bbf66d89SVitaly Kuznetsov if (fallback && cs == old_wd)
639bbf66d89SVitaly Kuznetsov continue;
640bbf66d89SVitaly Kuznetsov
641fb63a0ebSMartin Schwidefsky /* Pick the best watchdog. */
642bbf66d89SVitaly Kuznetsov if (!watchdog || cs->rating > watchdog->rating)
6435d8b34fdSThomas Gleixner watchdog = cs;
644bbf66d89SVitaly Kuznetsov }
645bbf66d89SVitaly Kuznetsov /* If we failed to find a fallback restore the old one. */
646bbf66d89SVitaly Kuznetsov if (!watchdog)
647bbf66d89SVitaly Kuznetsov watchdog = old_wd;
648bbf66d89SVitaly Kuznetsov
649bbf66d89SVitaly Kuznetsov /* If we changed the watchdog we need to reset cycles. */
650bbf66d89SVitaly Kuznetsov if (watchdog != old_wd)
6510f8e8ef7SMartin Schwidefsky clocksource_reset_watchdog();
652bbf66d89SVitaly Kuznetsov
653fb63a0ebSMartin Schwidefsky /* Check if the watchdog timer needs to be started. */
654fb63a0ebSMartin Schwidefsky clocksource_start_watchdog();
6555d8b34fdSThomas Gleixner spin_unlock_irqrestore(&watchdog_lock, flags);
6565d8b34fdSThomas Gleixner }
657fb63a0ebSMartin Schwidefsky
clocksource_dequeue_watchdog(struct clocksource * cs)658fb63a0ebSMartin Schwidefsky static void clocksource_dequeue_watchdog(struct clocksource *cs)
659fb63a0ebSMartin Schwidefsky {
660a89c7edbSThomas Gleixner if (cs != watchdog) {
661fb63a0ebSMartin Schwidefsky if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) {
662fb63a0ebSMartin Schwidefsky /* cs is a watched clocksource. */
663fb63a0ebSMartin Schwidefsky list_del_init(&cs->wd_list);
664fb63a0ebSMartin Schwidefsky /* Check if the watchdog timer needs to be stopped. */
665fb63a0ebSMartin Schwidefsky clocksource_stop_watchdog();
666a89c7edbSThomas Gleixner }
667a89c7edbSThomas Gleixner }
668fb63a0ebSMartin Schwidefsky }
669fb63a0ebSMartin Schwidefsky
__clocksource_watchdog_kthread(void)670e2c631baSPeter Zijlstra static int __clocksource_watchdog_kthread(void)
671c55c87c8SMartin Schwidefsky {
672c55c87c8SMartin Schwidefsky struct clocksource *cs, *tmp;
673c55c87c8SMartin Schwidefsky unsigned long flags;
674332962f2SThomas Gleixner int select = 0;
675c55c87c8SMartin Schwidefsky
6767560c02bSPaul E. McKenney /* Do any required per-CPU skew verification. */
6777560c02bSPaul E. McKenney if (curr_clocksource &&
6787560c02bSPaul E. McKenney curr_clocksource->flags & CLOCK_SOURCE_UNSTABLE &&
6797560c02bSPaul E. McKenney curr_clocksource->flags & CLOCK_SOURCE_VERIFY_PERCPU)
6807560c02bSPaul E. McKenney clocksource_verify_percpu(curr_clocksource);
6817560c02bSPaul E. McKenney
682c55c87c8SMartin Schwidefsky spin_lock_irqsave(&watchdog_lock, flags);
683332962f2SThomas Gleixner list_for_each_entry_safe(cs, tmp, &watchdog_list, wd_list) {
684c55c87c8SMartin Schwidefsky if (cs->flags & CLOCK_SOURCE_UNSTABLE) {
685c55c87c8SMartin Schwidefsky list_del_init(&cs->wd_list);
6862aae7bcfSPeter Zijlstra __clocksource_change_rating(cs, 0);
687332962f2SThomas Gleixner select = 1;
688332962f2SThomas Gleixner }
689332962f2SThomas Gleixner if (cs->flags & CLOCK_SOURCE_RESELECT) {
690332962f2SThomas Gleixner cs->flags &= ~CLOCK_SOURCE_RESELECT;
691332962f2SThomas Gleixner select = 1;
692332962f2SThomas Gleixner }
693c55c87c8SMartin Schwidefsky }
694c55c87c8SMartin Schwidefsky /* Check if the watchdog timer needs to be stopped. */
695c55c87c8SMartin Schwidefsky clocksource_stop_watchdog();
6966ea41d25SThomas Gleixner spin_unlock_irqrestore(&watchdog_lock, flags);
6976ea41d25SThomas Gleixner
698332962f2SThomas Gleixner return select;
699332962f2SThomas Gleixner }
700332962f2SThomas Gleixner
clocksource_watchdog_kthread(void * data)701e2c631baSPeter Zijlstra static int clocksource_watchdog_kthread(void *data)
702332962f2SThomas Gleixner {
703332962f2SThomas Gleixner mutex_lock(&clocksource_mutex);
704e2c631baSPeter Zijlstra if (__clocksource_watchdog_kthread())
705332962f2SThomas Gleixner clocksource_select();
706d0981a1bSThomas Gleixner mutex_unlock(&clocksource_mutex);
707e2c631baSPeter Zijlstra return 0;
708c55c87c8SMartin Schwidefsky }
709c55c87c8SMartin Schwidefsky
clocksource_is_watchdog(struct clocksource * cs)7107eaeb343SThomas Gleixner static bool clocksource_is_watchdog(struct clocksource *cs)
7117eaeb343SThomas Gleixner {
7127eaeb343SThomas Gleixner return cs == watchdog;
7137eaeb343SThomas Gleixner }
7147eaeb343SThomas Gleixner
715fb63a0ebSMartin Schwidefsky #else /* CONFIG_CLOCKSOURCE_WATCHDOG */
716fb63a0ebSMartin Schwidefsky
clocksource_enqueue_watchdog(struct clocksource * cs)717fb63a0ebSMartin Schwidefsky static void clocksource_enqueue_watchdog(struct clocksource *cs)
7185d8b34fdSThomas Gleixner {
7195d8b34fdSThomas Gleixner if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS)
7205d8b34fdSThomas Gleixner cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;
7215d8b34fdSThomas Gleixner }
722b52f52a0SThomas Gleixner
clocksource_select_watchdog(bool fallback)723bbf66d89SVitaly Kuznetsov static void clocksource_select_watchdog(bool fallback) { }
clocksource_dequeue_watchdog(struct clocksource * cs)724fb63a0ebSMartin Schwidefsky static inline void clocksource_dequeue_watchdog(struct clocksource *cs) { }
clocksource_resume_watchdog(void)725b52f52a0SThomas Gleixner static inline void clocksource_resume_watchdog(void) { }
__clocksource_watchdog_kthread(void)726e2c631baSPeter Zijlstra static inline int __clocksource_watchdog_kthread(void) { return 0; }
clocksource_is_watchdog(struct clocksource * cs)7277eaeb343SThomas Gleixner static bool clocksource_is_watchdog(struct clocksource *cs) { return false; }
clocksource_mark_unstable(struct clocksource * cs)728397bbf6dSPrarit Bhargava void clocksource_mark_unstable(struct clocksource *cs) { }
729fb63a0ebSMartin Schwidefsky
clocksource_watchdog_lock(unsigned long * flags)730db6f9e55SMathieu Malaterre static inline void clocksource_watchdog_lock(unsigned long *flags) { }
clocksource_watchdog_unlock(unsigned long * flags)731db6f9e55SMathieu Malaterre static inline void clocksource_watchdog_unlock(unsigned long *flags) { }
7322aae7bcfSPeter Zijlstra
733fb63a0ebSMartin Schwidefsky #endif /* CONFIG_CLOCKSOURCE_WATCHDOG */
7345d8b34fdSThomas Gleixner
clocksource_is_suspend(struct clocksource * cs)73539232ed5SBaolin Wang static bool clocksource_is_suspend(struct clocksource *cs)
73639232ed5SBaolin Wang {
73739232ed5SBaolin Wang return cs == suspend_clocksource;
73839232ed5SBaolin Wang }
73939232ed5SBaolin Wang
__clocksource_suspend_select(struct clocksource * cs)74039232ed5SBaolin Wang static void __clocksource_suspend_select(struct clocksource *cs)
74139232ed5SBaolin Wang {
74239232ed5SBaolin Wang /*
74339232ed5SBaolin Wang * Skip the clocksource which will be stopped in suspend state.
74439232ed5SBaolin Wang */
74539232ed5SBaolin Wang if (!(cs->flags & CLOCK_SOURCE_SUSPEND_NONSTOP))
74639232ed5SBaolin Wang return;
74739232ed5SBaolin Wang
74839232ed5SBaolin Wang /*
74939232ed5SBaolin Wang * The nonstop clocksource can be selected as the suspend clocksource to
75039232ed5SBaolin Wang * calculate the suspend time, so it should not supply suspend/resume
75139232ed5SBaolin Wang * interfaces to suspend the nonstop clocksource when system suspends.
75239232ed5SBaolin Wang */
75339232ed5SBaolin Wang if (cs->suspend || cs->resume) {
75439232ed5SBaolin Wang pr_warn("Nonstop clocksource %s should not supply suspend/resume interfaces\n",
75539232ed5SBaolin Wang cs->name);
75639232ed5SBaolin Wang }
75739232ed5SBaolin Wang
75839232ed5SBaolin Wang /* Pick the best rating. */
75939232ed5SBaolin Wang if (!suspend_clocksource || cs->rating > suspend_clocksource->rating)
76039232ed5SBaolin Wang suspend_clocksource = cs;
76139232ed5SBaolin Wang }
76239232ed5SBaolin Wang
76339232ed5SBaolin Wang /**
76439232ed5SBaolin Wang * clocksource_suspend_select - Select the best clocksource for suspend timing
76539232ed5SBaolin Wang * @fallback: if select a fallback clocksource
76639232ed5SBaolin Wang */
clocksource_suspend_select(bool fallback)76739232ed5SBaolin Wang static void clocksource_suspend_select(bool fallback)
76839232ed5SBaolin Wang {
76939232ed5SBaolin Wang struct clocksource *cs, *old_suspend;
77039232ed5SBaolin Wang
77139232ed5SBaolin Wang old_suspend = suspend_clocksource;
77239232ed5SBaolin Wang if (fallback)
77339232ed5SBaolin Wang suspend_clocksource = NULL;
77439232ed5SBaolin Wang
77539232ed5SBaolin Wang list_for_each_entry(cs, &clocksource_list, list) {
77639232ed5SBaolin Wang /* Skip current if we were requested for a fallback. */
77739232ed5SBaolin Wang if (fallback && cs == old_suspend)
77839232ed5SBaolin Wang continue;
77939232ed5SBaolin Wang
78039232ed5SBaolin Wang __clocksource_suspend_select(cs);
78139232ed5SBaolin Wang }
78239232ed5SBaolin Wang }
78339232ed5SBaolin Wang
78439232ed5SBaolin Wang /**
78539232ed5SBaolin Wang * clocksource_start_suspend_timing - Start measuring the suspend timing
78639232ed5SBaolin Wang * @cs: current clocksource from timekeeping
78739232ed5SBaolin Wang * @start_cycles: current cycles from timekeeping
78839232ed5SBaolin Wang *
78939232ed5SBaolin Wang * This function will save the start cycle values of suspend timer to calculate
79039232ed5SBaolin Wang * the suspend time when resuming system.
79139232ed5SBaolin Wang *
79239232ed5SBaolin Wang * This function is called late in the suspend process from timekeeping_suspend(),
7934bf07f65SIngo Molnar * that means processes are frozen, non-boot cpus and interrupts are disabled
79439232ed5SBaolin Wang * now. It is therefore possible to start the suspend timer without taking the
79539232ed5SBaolin Wang * clocksource mutex.
79639232ed5SBaolin Wang */
clocksource_start_suspend_timing(struct clocksource * cs,u64 start_cycles)79739232ed5SBaolin Wang void clocksource_start_suspend_timing(struct clocksource *cs, u64 start_cycles)
79839232ed5SBaolin Wang {
79939232ed5SBaolin Wang if (!suspend_clocksource)
80039232ed5SBaolin Wang return;
80139232ed5SBaolin Wang
80239232ed5SBaolin Wang /*
80339232ed5SBaolin Wang * If current clocksource is the suspend timer, we should use the
80439232ed5SBaolin Wang * tkr_mono.cycle_last value as suspend_start to avoid same reading
80539232ed5SBaolin Wang * from suspend timer.
80639232ed5SBaolin Wang */
80739232ed5SBaolin Wang if (clocksource_is_suspend(cs)) {
80839232ed5SBaolin Wang suspend_start = start_cycles;
80939232ed5SBaolin Wang return;
81039232ed5SBaolin Wang }
81139232ed5SBaolin Wang
81239232ed5SBaolin Wang if (suspend_clocksource->enable &&
81339232ed5SBaolin Wang suspend_clocksource->enable(suspend_clocksource)) {
81439232ed5SBaolin Wang pr_warn_once("Failed to enable the non-suspend-able clocksource.\n");
81539232ed5SBaolin Wang return;
81639232ed5SBaolin Wang }
81739232ed5SBaolin Wang
81839232ed5SBaolin Wang suspend_start = suspend_clocksource->read(suspend_clocksource);
81939232ed5SBaolin Wang }
82039232ed5SBaolin Wang
82139232ed5SBaolin Wang /**
82239232ed5SBaolin Wang * clocksource_stop_suspend_timing - Stop measuring the suspend timing
82339232ed5SBaolin Wang * @cs: current clocksource from timekeeping
82439232ed5SBaolin Wang * @cycle_now: current cycles from timekeeping
82539232ed5SBaolin Wang *
82639232ed5SBaolin Wang * This function will calculate the suspend time from suspend timer.
82739232ed5SBaolin Wang *
82839232ed5SBaolin Wang * Returns nanoseconds since suspend started, 0 if no usable suspend clocksource.
82939232ed5SBaolin Wang *
83039232ed5SBaolin Wang * This function is called early in the resume process from timekeeping_resume(),
83139232ed5SBaolin Wang * that means there is only one cpu, no processes are running and the interrupts
83239232ed5SBaolin Wang * are disabled. It is therefore possible to stop the suspend timer without
83339232ed5SBaolin Wang * taking the clocksource mutex.
83439232ed5SBaolin Wang */
clocksource_stop_suspend_timing(struct clocksource * cs,u64 cycle_now)83539232ed5SBaolin Wang u64 clocksource_stop_suspend_timing(struct clocksource *cs, u64 cycle_now)
83639232ed5SBaolin Wang {
83739232ed5SBaolin Wang u64 now, delta, nsec = 0;
83839232ed5SBaolin Wang
83939232ed5SBaolin Wang if (!suspend_clocksource)
84039232ed5SBaolin Wang return 0;
84139232ed5SBaolin Wang
84239232ed5SBaolin Wang /*
84339232ed5SBaolin Wang * If current clocksource is the suspend timer, we should use the
84439232ed5SBaolin Wang * tkr_mono.cycle_last value from timekeeping as current cycle to
84539232ed5SBaolin Wang * avoid same reading from suspend timer.
84639232ed5SBaolin Wang */
84739232ed5SBaolin Wang if (clocksource_is_suspend(cs))
84839232ed5SBaolin Wang now = cycle_now;
84939232ed5SBaolin Wang else
85039232ed5SBaolin Wang now = suspend_clocksource->read(suspend_clocksource);
85139232ed5SBaolin Wang
85239232ed5SBaolin Wang if (now > suspend_start) {
85339232ed5SBaolin Wang delta = clocksource_delta(now, suspend_start,
85439232ed5SBaolin Wang suspend_clocksource->mask);
85539232ed5SBaolin Wang nsec = mul_u64_u32_shr(delta, suspend_clocksource->mult,
85639232ed5SBaolin Wang suspend_clocksource->shift);
85739232ed5SBaolin Wang }
85839232ed5SBaolin Wang
85939232ed5SBaolin Wang /*
86039232ed5SBaolin Wang * Disable the suspend timer to save power if current clocksource is
86139232ed5SBaolin Wang * not the suspend timer.
86239232ed5SBaolin Wang */
86339232ed5SBaolin Wang if (!clocksource_is_suspend(cs) && suspend_clocksource->disable)
86439232ed5SBaolin Wang suspend_clocksource->disable(suspend_clocksource);
86539232ed5SBaolin Wang
86639232ed5SBaolin Wang return nsec;
86739232ed5SBaolin Wang }
86839232ed5SBaolin Wang
869734efb46Sjohn stultz /**
870c54a42b1SMagnus Damm * clocksource_suspend - suspend the clocksource(s)
871c54a42b1SMagnus Damm */
clocksource_suspend(void)872c54a42b1SMagnus Damm void clocksource_suspend(void)
873c54a42b1SMagnus Damm {
874c54a42b1SMagnus Damm struct clocksource *cs;
875c54a42b1SMagnus Damm
876c54a42b1SMagnus Damm list_for_each_entry_reverse(cs, &clocksource_list, list)
877c54a42b1SMagnus Damm if (cs->suspend)
878c54a42b1SMagnus Damm cs->suspend(cs);
879c54a42b1SMagnus Damm }
880c54a42b1SMagnus Damm
881c54a42b1SMagnus Damm /**
882b52f52a0SThomas Gleixner * clocksource_resume - resume the clocksource(s)
883b52f52a0SThomas Gleixner */
clocksource_resume(void)884b52f52a0SThomas Gleixner void clocksource_resume(void)
885b52f52a0SThomas Gleixner {
8862e197586SMatthias Kaehlcke struct clocksource *cs;
887b52f52a0SThomas Gleixner
88875c5158fSMartin Schwidefsky list_for_each_entry(cs, &clocksource_list, list)
889b52f52a0SThomas Gleixner if (cs->resume)
89017622339SMagnus Damm cs->resume(cs);
891b52f52a0SThomas Gleixner
892b52f52a0SThomas Gleixner clocksource_resume_watchdog();
893b52f52a0SThomas Gleixner }
894b52f52a0SThomas Gleixner
895b52f52a0SThomas Gleixner /**
8967c3078b6SJason Wessel * clocksource_touch_watchdog - Update watchdog
8977c3078b6SJason Wessel *
8987c3078b6SJason Wessel * Update the watchdog after exception contexts such as kgdb so as not
8997b7422a5SThomas Gleixner * to incorrectly trip the watchdog. This might fail when the kernel
9007b7422a5SThomas Gleixner * was stopped in code which holds watchdog_lock.
9017c3078b6SJason Wessel */
clocksource_touch_watchdog(void)9027c3078b6SJason Wessel void clocksource_touch_watchdog(void)
9037c3078b6SJason Wessel {
9047c3078b6SJason Wessel clocksource_resume_watchdog();
9057c3078b6SJason Wessel }
9067c3078b6SJason Wessel
907734efb46Sjohn stultz /**
908d65670a7SJohn Stultz * clocksource_max_adjustment- Returns max adjustment amount
909d65670a7SJohn Stultz * @cs: Pointer to clocksource
910d65670a7SJohn Stultz *
911d65670a7SJohn Stultz */
clocksource_max_adjustment(struct clocksource * cs)912d65670a7SJohn Stultz static u32 clocksource_max_adjustment(struct clocksource *cs)
913d65670a7SJohn Stultz {
914d65670a7SJohn Stultz u64 ret;
915d65670a7SJohn Stultz /*
91688b28adfSJim Cromie * We won't try to correct for more than 11% adjustments (110,000 ppm),
917d65670a7SJohn Stultz */
918d65670a7SJohn Stultz ret = (u64)cs->mult * 11;
919d65670a7SJohn Stultz do_div(ret,100);
920d65670a7SJohn Stultz return (u32)ret;
921d65670a7SJohn Stultz }
922d65670a7SJohn Stultz
923d65670a7SJohn Stultz /**
92487d8b9ebSStephen Boyd * clocks_calc_max_nsecs - Returns maximum nanoseconds that can be converted
92587d8b9ebSStephen Boyd * @mult: cycle to nanosecond multiplier
92687d8b9ebSStephen Boyd * @shift: cycle to nanosecond divisor (power of two)
92787d8b9ebSStephen Boyd * @maxadj: maximum adjustment value to mult (~11%)
92887d8b9ebSStephen Boyd * @mask: bitmask for two's complement subtraction of non 64 bit counters
929fb82fe2fSJohn Stultz * @max_cyc: maximum cycle value before potential overflow (does not include
930fb82fe2fSJohn Stultz * any safety margin)
931362fde04SJohn Stultz *
9328e56f33fSJohn Stultz * NOTE: This function includes a safety margin of 50%, in other words, we
9338e56f33fSJohn Stultz * return half the number of nanoseconds the hardware counter can technically
9348e56f33fSJohn Stultz * cover. This is done so that we can potentially detect problems caused by
9358e56f33fSJohn Stultz * delayed timers or bad hardware, which might result in time intervals that
936571af55aSZhen Lei * are larger than what the math used can handle without overflows.
93798962465SJon Hunter */
clocks_calc_max_nsecs(u32 mult,u32 shift,u32 maxadj,u64 mask,u64 * max_cyc)938fb82fe2fSJohn Stultz u64 clocks_calc_max_nsecs(u32 mult, u32 shift, u32 maxadj, u64 mask, u64 *max_cyc)
93998962465SJon Hunter {
94098962465SJon Hunter u64 max_nsecs, max_cycles;
94198962465SJon Hunter
94298962465SJon Hunter /*
94398962465SJon Hunter * Calculate the maximum number of cycles that we can pass to the
9446086e346SJohn Stultz * cyc2ns() function without overflowing a 64-bit result.
94598962465SJon Hunter */
9466086e346SJohn Stultz max_cycles = ULLONG_MAX;
9476086e346SJohn Stultz do_div(max_cycles, mult+maxadj);
94898962465SJon Hunter
94998962465SJon Hunter /*
95098962465SJon Hunter * The actual maximum number of cycles we can defer the clocksource is
95187d8b9ebSStephen Boyd * determined by the minimum of max_cycles and mask.
952d65670a7SJohn Stultz * Note: Here we subtract the maxadj to make sure we don't sleep for
953d65670a7SJohn Stultz * too long if there's a large negative adjustment.
95498962465SJon Hunter */
95587d8b9ebSStephen Boyd max_cycles = min(max_cycles, mask);
95687d8b9ebSStephen Boyd max_nsecs = clocksource_cyc2ns(max_cycles, mult - maxadj, shift);
95798962465SJon Hunter
958fb82fe2fSJohn Stultz /* return the max_cycles value as well if requested */
959fb82fe2fSJohn Stultz if (max_cyc)
960fb82fe2fSJohn Stultz *max_cyc = max_cycles;
961fb82fe2fSJohn Stultz
962362fde04SJohn Stultz /* Return 50% of the actual maximum, so we can detect bad values */
963362fde04SJohn Stultz max_nsecs >>= 1;
964362fde04SJohn Stultz
96587d8b9ebSStephen Boyd return max_nsecs;
96687d8b9ebSStephen Boyd }
96787d8b9ebSStephen Boyd
96887d8b9ebSStephen Boyd /**
969fb82fe2fSJohn Stultz * clocksource_update_max_deferment - Updates the clocksource max_idle_ns & max_cycles
970fb82fe2fSJohn Stultz * @cs: Pointer to clocksource to be updated
97187d8b9ebSStephen Boyd *
97287d8b9ebSStephen Boyd */
clocksource_update_max_deferment(struct clocksource * cs)973fb82fe2fSJohn Stultz static inline void clocksource_update_max_deferment(struct clocksource *cs)
97487d8b9ebSStephen Boyd {
975fb82fe2fSJohn Stultz cs->max_idle_ns = clocks_calc_max_nsecs(cs->mult, cs->shift,
976fb82fe2fSJohn Stultz cs->maxadj, cs->mask,
977fb82fe2fSJohn Stultz &cs->max_cycles);
97898962465SJon Hunter }
97998962465SJon Hunter
clocksource_find_best(bool oneshot,bool skipcur)980f5a2e343SThomas Gleixner static struct clocksource *clocksource_find_best(bool oneshot, bool skipcur)
9815d33b883SThomas Gleixner {
9825d33b883SThomas Gleixner struct clocksource *cs;
9835d33b883SThomas Gleixner
9845d33b883SThomas Gleixner if (!finished_booting || list_empty(&clocksource_list))
9855d33b883SThomas Gleixner return NULL;
9865d33b883SThomas Gleixner
9875d33b883SThomas Gleixner /*
9885d33b883SThomas Gleixner * We pick the clocksource with the highest rating. If oneshot
9895d33b883SThomas Gleixner * mode is active, we pick the highres valid clocksource with
9905d33b883SThomas Gleixner * the best rating.
9915d33b883SThomas Gleixner */
9925d33b883SThomas Gleixner list_for_each_entry(cs, &clocksource_list, list) {
993f5a2e343SThomas Gleixner if (skipcur && cs == curr_clocksource)
994f5a2e343SThomas Gleixner continue;
9955d33b883SThomas Gleixner if (oneshot && !(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES))
9965d33b883SThomas Gleixner continue;
9975d33b883SThomas Gleixner return cs;
9985d33b883SThomas Gleixner }
9995d33b883SThomas Gleixner return NULL;
10005d33b883SThomas Gleixner }
10015d33b883SThomas Gleixner
__clocksource_select(bool skipcur)1002f5a2e343SThomas Gleixner static void __clocksource_select(bool skipcur)
1003734efb46Sjohn stultz {
10045d33b883SThomas Gleixner bool oneshot = tick_oneshot_mode_active();
1005f1b82746SMartin Schwidefsky struct clocksource *best, *cs;
10065d8b34fdSThomas Gleixner
10075d33b883SThomas Gleixner /* Find the best suitable clocksource */
1008f5a2e343SThomas Gleixner best = clocksource_find_best(oneshot, skipcur);
10095d33b883SThomas Gleixner if (!best)
1010f1b82746SMartin Schwidefsky return;
10115d33b883SThomas Gleixner
10127f852afeSBaolin Wang if (!strlen(override_name))
10137f852afeSBaolin Wang goto found;
10147f852afeSBaolin Wang
1015f1b82746SMartin Schwidefsky /* Check for the override clocksource. */
1016f1b82746SMartin Schwidefsky list_for_each_entry(cs, &clocksource_list, list) {
1017f5a2e343SThomas Gleixner if (skipcur && cs == curr_clocksource)
1018f5a2e343SThomas Gleixner continue;
1019f1b82746SMartin Schwidefsky if (strcmp(cs->name, override_name) != 0)
1020f1b82746SMartin Schwidefsky continue;
1021f1b82746SMartin Schwidefsky /*
1022f1b82746SMartin Schwidefsky * Check to make sure we don't switch to a non-highres
1023f1b82746SMartin Schwidefsky * capable clocksource if the tick code is in oneshot
1024f1b82746SMartin Schwidefsky * mode (highres or nohz)
1025f1b82746SMartin Schwidefsky */
10265d33b883SThomas Gleixner if (!(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) && oneshot) {
1027f1b82746SMartin Schwidefsky /* Override clocksource cannot be used. */
102836374583SKyle Walker if (cs->flags & CLOCK_SOURCE_UNSTABLE) {
102936374583SKyle Walker pr_warn("Override clocksource %s is unstable and not HRT compatible - cannot switch while in HRT/NOHZ mode\n",
103045bbfe64SJoe Perches cs->name);
1031f1b82746SMartin Schwidefsky override_name[0] = 0;
103236374583SKyle Walker } else {
103336374583SKyle Walker /*
103436374583SKyle Walker * The override cannot be currently verified.
103536374583SKyle Walker * Deferring to let the watchdog check.
103636374583SKyle Walker */
103736374583SKyle Walker pr_info("Override clocksource %s is not currently HRT compatible - deferring\n",
103836374583SKyle Walker cs->name);
103936374583SKyle Walker }
1040f1b82746SMartin Schwidefsky } else
1041f1b82746SMartin Schwidefsky /* Override clocksource can be used. */
1042f1b82746SMartin Schwidefsky best = cs;
1043f1b82746SMartin Schwidefsky break;
1044734efb46Sjohn stultz }
1045ba919d1cSThomas Gleixner
10467f852afeSBaolin Wang found:
1047ba919d1cSThomas Gleixner if (curr_clocksource != best && !timekeeping_notify(best)) {
1048ba919d1cSThomas Gleixner pr_info("Switched to clocksource %s\n", best->name);
104975c5158fSMartin Schwidefsky curr_clocksource = best;
1050f1b82746SMartin Schwidefsky }
105175c5158fSMartin Schwidefsky }
105275c5158fSMartin Schwidefsky
1053f5a2e343SThomas Gleixner /**
1054f5a2e343SThomas Gleixner * clocksource_select - Select the best clocksource available
1055f5a2e343SThomas Gleixner *
1056f5a2e343SThomas Gleixner * Private function. Must hold clocksource_mutex when called.
1057f5a2e343SThomas Gleixner *
1058f5a2e343SThomas Gleixner * Select the clocksource with the best rating, or the clocksource,
1059f5a2e343SThomas Gleixner * which is selected by userspace override.
1060f5a2e343SThomas Gleixner */
clocksource_select(void)1061f5a2e343SThomas Gleixner static void clocksource_select(void)
1062f5a2e343SThomas Gleixner {
1063cfed432dSGuillaume Gomez __clocksource_select(false);
1064f5a2e343SThomas Gleixner }
1065f5a2e343SThomas Gleixner
clocksource_select_fallback(void)10667eaeb343SThomas Gleixner static void clocksource_select_fallback(void)
10677eaeb343SThomas Gleixner {
1068cfed432dSGuillaume Gomez __clocksource_select(true);
10697eaeb343SThomas Gleixner }
10707eaeb343SThomas Gleixner
107175c5158fSMartin Schwidefsky /*
107275c5158fSMartin Schwidefsky * clocksource_done_booting - Called near the end of core bootup
107375c5158fSMartin Schwidefsky *
107475c5158fSMartin Schwidefsky * Hack to avoid lots of clocksource churn at boot time.
107575c5158fSMartin Schwidefsky * We use fs_initcall because we want this to start before
107675c5158fSMartin Schwidefsky * device_initcall but after subsys_initcall.
107775c5158fSMartin Schwidefsky */
clocksource_done_booting(void)107875c5158fSMartin Schwidefsky static int __init clocksource_done_booting(void)
107975c5158fSMartin Schwidefsky {
1080ad6759fbSjohn stultz mutex_lock(&clocksource_mutex);
1081ad6759fbSjohn stultz curr_clocksource = clocksource_default_clock();
108275c5158fSMartin Schwidefsky finished_booting = 1;
108354a6bc0bSThomas Gleixner /*
108454a6bc0bSThomas Gleixner * Run the watchdog first to eliminate unstable clock sources
108554a6bc0bSThomas Gleixner */
1086e2c631baSPeter Zijlstra __clocksource_watchdog_kthread();
108775c5158fSMartin Schwidefsky clocksource_select();
1088e6c73305SThomas Gleixner mutex_unlock(&clocksource_mutex);
108975c5158fSMartin Schwidefsky return 0;
109075c5158fSMartin Schwidefsky }
109175c5158fSMartin Schwidefsky fs_initcall(clocksource_done_booting);
1092f1b82746SMartin Schwidefsky
109392c7e002SThomas Gleixner /*
109492c7e002SThomas Gleixner * Enqueue the clocksource sorted by rating
1095734efb46Sjohn stultz */
clocksource_enqueue(struct clocksource * cs)1096f1b82746SMartin Schwidefsky static void clocksource_enqueue(struct clocksource *cs)
1097734efb46Sjohn stultz {
1098f1b82746SMartin Schwidefsky struct list_head *entry = &clocksource_list;
1099f1b82746SMartin Schwidefsky struct clocksource *tmp;
1100734efb46Sjohn stultz
11010fb71d34SMinfei Huang list_for_each_entry(tmp, &clocksource_list, list) {
110292c7e002SThomas Gleixner /* Keep track of the place, where to insert */
11030fb71d34SMinfei Huang if (tmp->rating < cs->rating)
11040fb71d34SMinfei Huang break;
1105f1b82746SMartin Schwidefsky entry = &tmp->list;
11060fb71d34SMinfei Huang }
1107f1b82746SMartin Schwidefsky list_add(&cs->list, entry);
1108734efb46Sjohn stultz }
1109734efb46Sjohn stultz
1110d7e81c26SJohn Stultz /**
1111fba9e072SJohn Stultz * __clocksource_update_freq_scale - Used update clocksource with new freq
1112b1b73d09SKusanagi Kouichi * @cs: clocksource to be registered
1113852db46dSJohn Stultz * @scale: Scale factor multiplied against freq to get clocksource hz
1114852db46dSJohn Stultz * @freq: clocksource frequency (cycles per second) divided by scale
1115852db46dSJohn Stultz *
1116852db46dSJohn Stultz * This should only be called from the clocksource->enable() method.
1117852db46dSJohn Stultz *
1118852db46dSJohn Stultz * This *SHOULD NOT* be called directly! Please use the
1119fba9e072SJohn Stultz * __clocksource_update_freq_hz() or __clocksource_update_freq_khz() helper
1120fba9e072SJohn Stultz * functions.
1121852db46dSJohn Stultz */
__clocksource_update_freq_scale(struct clocksource * cs,u32 scale,u32 freq)1122fba9e072SJohn Stultz void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq)
1123852db46dSJohn Stultz {
1124c0e299b1SThomas Gleixner u64 sec;
1125f8935983SJohn Stultz
1126f8935983SJohn Stultz /*
1127f8935983SJohn Stultz * Default clocksources are *special* and self-define their mult/shift.
1128f8935983SJohn Stultz * But, you're not special, so you should specify a freq value.
1129f8935983SJohn Stultz */
1130f8935983SJohn Stultz if (freq) {
1131852db46dSJohn Stultz /*
1132724ed53eSThomas Gleixner * Calc the maximum number of seconds which we can run before
1133f8935983SJohn Stultz * wrapping around. For clocksources which have a mask > 32-bit
1134724ed53eSThomas Gleixner * we need to limit the max sleep time to have a good
1135724ed53eSThomas Gleixner * conversion precision. 10 minutes is still a reasonable
1136724ed53eSThomas Gleixner * amount. That results in a shift value of 24 for a
1137f8935983SJohn Stultz * clocksource with mask >= 40-bit and f >= 4GHz. That maps to
1138362fde04SJohn Stultz * ~ 0.06ppm granularity for NTP.
1139852db46dSJohn Stultz */
1140362fde04SJohn Stultz sec = cs->mask;
1141724ed53eSThomas Gleixner do_div(sec, freq);
1142724ed53eSThomas Gleixner do_div(sec, scale);
1143724ed53eSThomas Gleixner if (!sec)
1144724ed53eSThomas Gleixner sec = 1;
1145724ed53eSThomas Gleixner else if (sec > 600 && cs->mask > UINT_MAX)
1146724ed53eSThomas Gleixner sec = 600;
1147724ed53eSThomas Gleixner
1148852db46dSJohn Stultz clocks_calc_mult_shift(&cs->mult, &cs->shift, freq,
1149724ed53eSThomas Gleixner NSEC_PER_SEC / scale, sec * scale);
1150f8935983SJohn Stultz }
11512e27e793SPaul E. McKenney
11522e27e793SPaul E. McKenney /*
11532e27e793SPaul E. McKenney * If the uncertainty margin is not specified, calculate it.
11542e27e793SPaul E. McKenney * If both scale and freq are non-zero, calculate the clock
11552e27e793SPaul E. McKenney * period, but bound below at 2*WATCHDOG_MAX_SKEW. However,
11562e27e793SPaul E. McKenney * if either of scale or freq is zero, be very conservative and
11572e27e793SPaul E. McKenney * take the tens-of-milliseconds WATCHDOG_THRESHOLD value for the
11582e27e793SPaul E. McKenney * uncertainty margin. Allow stupidly small uncertainty margins
11592e27e793SPaul E. McKenney * to be specified by the caller for testing purposes, but warn
11602e27e793SPaul E. McKenney * to discourage production use of this capability.
11612e27e793SPaul E. McKenney */
11622e27e793SPaul E. McKenney if (scale && freq && !cs->uncertainty_margin) {
11632e27e793SPaul E. McKenney cs->uncertainty_margin = NSEC_PER_SEC / (scale * freq);
11642e27e793SPaul E. McKenney if (cs->uncertainty_margin < 2 * WATCHDOG_MAX_SKEW)
11652e27e793SPaul E. McKenney cs->uncertainty_margin = 2 * WATCHDOG_MAX_SKEW;
11662e27e793SPaul E. McKenney } else if (!cs->uncertainty_margin) {
11672e27e793SPaul E. McKenney cs->uncertainty_margin = WATCHDOG_THRESHOLD;
11682e27e793SPaul E. McKenney }
11692e27e793SPaul E. McKenney WARN_ON_ONCE(cs->uncertainty_margin < 2 * WATCHDOG_MAX_SKEW);
11702e27e793SPaul E. McKenney
1171d65670a7SJohn Stultz /*
1172362fde04SJohn Stultz * Ensure clocksources that have large 'mult' values don't overflow
1173362fde04SJohn Stultz * when adjusted.
1174d65670a7SJohn Stultz */
1175d65670a7SJohn Stultz cs->maxadj = clocksource_max_adjustment(cs);
1176f8935983SJohn Stultz while (freq && ((cs->mult + cs->maxadj < cs->mult)
1177f8935983SJohn Stultz || (cs->mult - cs->maxadj > cs->mult))) {
1178d65670a7SJohn Stultz cs->mult >>= 1;
1179d65670a7SJohn Stultz cs->shift--;
1180d65670a7SJohn Stultz cs->maxadj = clocksource_max_adjustment(cs);
1181d65670a7SJohn Stultz }
1182d65670a7SJohn Stultz
1183f8935983SJohn Stultz /*
1184f8935983SJohn Stultz * Only warn for *special* clocksources that self-define
1185f8935983SJohn Stultz * their mult/shift values and don't specify a freq.
1186f8935983SJohn Stultz */
1187f8935983SJohn Stultz WARN_ONCE(cs->mult + cs->maxadj < cs->mult,
1188f8935983SJohn Stultz "timekeeping: Clocksource %s might overflow on 11%% adjustment\n",
1189f8935983SJohn Stultz cs->name);
1190f8935983SJohn Stultz
1191fb82fe2fSJohn Stultz clocksource_update_max_deferment(cs);
11928cc8c525SJohn Stultz
119345bbfe64SJoe Perches pr_info("%s: mask: 0x%llx max_cycles: 0x%llx, max_idle_ns: %lld ns\n",
11948cc8c525SJohn Stultz cs->name, cs->mask, cs->max_cycles, cs->max_idle_ns);
1195852db46dSJohn Stultz }
1196fba9e072SJohn Stultz EXPORT_SYMBOL_GPL(__clocksource_update_freq_scale);
1197852db46dSJohn Stultz
1198852db46dSJohn Stultz /**
1199d7e81c26SJohn Stultz * __clocksource_register_scale - Used to install new clocksources
1200b1b73d09SKusanagi Kouichi * @cs: clocksource to be registered
1201d7e81c26SJohn Stultz * @scale: Scale factor multiplied against freq to get clocksource hz
1202d7e81c26SJohn Stultz * @freq: clocksource frequency (cycles per second) divided by scale
1203d7e81c26SJohn Stultz *
1204d7e81c26SJohn Stultz * Returns -EBUSY if registration fails, zero otherwise.
1205d7e81c26SJohn Stultz *
1206d7e81c26SJohn Stultz * This *SHOULD NOT* be called directly! Please use the
1207d7e81c26SJohn Stultz * clocksource_register_hz() or clocksource_register_khz helper functions.
1208d7e81c26SJohn Stultz */
__clocksource_register_scale(struct clocksource * cs,u32 scale,u32 freq)1209d7e81c26SJohn Stultz int __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq)
1210d7e81c26SJohn Stultz {
12112aae7bcfSPeter Zijlstra unsigned long flags;
1212d7e81c26SJohn Stultz
1213d67f34c1SThomas Gleixner clocksource_arch_init(cs);
1214d67f34c1SThomas Gleixner
1215b2c67cbeSThomas Gleixner if (WARN_ON_ONCE((unsigned int)cs->id >= CSID_MAX))
1216b2c67cbeSThomas Gleixner cs->id = CSID_GENERIC;
12175d51bee7SThomas Gleixner if (cs->vdso_clock_mode < 0 ||
12185d51bee7SThomas Gleixner cs->vdso_clock_mode >= VDSO_CLOCKMODE_MAX) {
12195d51bee7SThomas Gleixner pr_warn("clocksource %s registered with invalid VDSO mode %d. Disabling VDSO support.\n",
12205d51bee7SThomas Gleixner cs->name, cs->vdso_clock_mode);
12215d51bee7SThomas Gleixner cs->vdso_clock_mode = VDSO_CLOCKMODE_NONE;
12225d51bee7SThomas Gleixner }
12235d51bee7SThomas Gleixner
1224b595076aSUwe Kleine-König /* Initialize mult/shift and max_idle_ns */
1225fba9e072SJohn Stultz __clocksource_update_freq_scale(cs, scale, freq);
1226d7e81c26SJohn Stultz
1227be278e98SJames Hartley /* Add clocksource to the clocksource list */
1228d7e81c26SJohn Stultz mutex_lock(&clocksource_mutex);
12292aae7bcfSPeter Zijlstra
12302aae7bcfSPeter Zijlstra clocksource_watchdog_lock(&flags);
1231d7e81c26SJohn Stultz clocksource_enqueue(cs);
1232d7e81c26SJohn Stultz clocksource_enqueue_watchdog(cs);
12332aae7bcfSPeter Zijlstra clocksource_watchdog_unlock(&flags);
12342aae7bcfSPeter Zijlstra
1235e05b2efbSjohn stultz clocksource_select();
1236bbf66d89SVitaly Kuznetsov clocksource_select_watchdog(false);
123739232ed5SBaolin Wang __clocksource_suspend_select(cs);
1238d7e81c26SJohn Stultz mutex_unlock(&clocksource_mutex);
1239d7e81c26SJohn Stultz return 0;
1240d7e81c26SJohn Stultz }
1241d7e81c26SJohn Stultz EXPORT_SYMBOL_GPL(__clocksource_register_scale);
1242d7e81c26SJohn Stultz
__clocksource_change_rating(struct clocksource * cs,int rating)1243d0981a1bSThomas Gleixner static void __clocksource_change_rating(struct clocksource *cs, int rating)
1244d0981a1bSThomas Gleixner {
1245d0981a1bSThomas Gleixner list_del(&cs->list);
1246d0981a1bSThomas Gleixner cs->rating = rating;
1247d0981a1bSThomas Gleixner clocksource_enqueue(cs);
1248d0981a1bSThomas Gleixner }
1249d0981a1bSThomas Gleixner
1250734efb46Sjohn stultz /**
125192c7e002SThomas Gleixner * clocksource_change_rating - Change the rating of a registered clocksource
1252b1b73d09SKusanagi Kouichi * @cs: clocksource to be changed
1253b1b73d09SKusanagi Kouichi * @rating: new rating
1254734efb46Sjohn stultz */
clocksource_change_rating(struct clocksource * cs,int rating)125592c7e002SThomas Gleixner void clocksource_change_rating(struct clocksource *cs, int rating)
1256734efb46Sjohn stultz {
12572aae7bcfSPeter Zijlstra unsigned long flags;
12582aae7bcfSPeter Zijlstra
125975c5158fSMartin Schwidefsky mutex_lock(&clocksource_mutex);
12602aae7bcfSPeter Zijlstra clocksource_watchdog_lock(&flags);
1261d0981a1bSThomas Gleixner __clocksource_change_rating(cs, rating);
12622aae7bcfSPeter Zijlstra clocksource_watchdog_unlock(&flags);
12632aae7bcfSPeter Zijlstra
1264332962f2SThomas Gleixner clocksource_select();
1265bbf66d89SVitaly Kuznetsov clocksource_select_watchdog(false);
126639232ed5SBaolin Wang clocksource_suspend_select(false);
126775c5158fSMartin Schwidefsky mutex_unlock(&clocksource_mutex);
1268734efb46Sjohn stultz }
1269fb63a0ebSMartin Schwidefsky EXPORT_SYMBOL(clocksource_change_rating);
1270734efb46Sjohn stultz
12717eaeb343SThomas Gleixner /*
12727eaeb343SThomas Gleixner * Unbind clocksource @cs. Called with clocksource_mutex held
12737eaeb343SThomas Gleixner */
clocksource_unbind(struct clocksource * cs)12747eaeb343SThomas Gleixner static int clocksource_unbind(struct clocksource *cs)
12757eaeb343SThomas Gleixner {
12762aae7bcfSPeter Zijlstra unsigned long flags;
12772aae7bcfSPeter Zijlstra
1278bbf66d89SVitaly Kuznetsov if (clocksource_is_watchdog(cs)) {
1279bbf66d89SVitaly Kuznetsov /* Select and try to install a replacement watchdog. */
1280bbf66d89SVitaly Kuznetsov clocksource_select_watchdog(true);
12817eaeb343SThomas Gleixner if (clocksource_is_watchdog(cs))
12827eaeb343SThomas Gleixner return -EBUSY;
1283bbf66d89SVitaly Kuznetsov }
12847eaeb343SThomas Gleixner
12857eaeb343SThomas Gleixner if (cs == curr_clocksource) {
12867eaeb343SThomas Gleixner /* Select and try to install a replacement clock source */
12877eaeb343SThomas Gleixner clocksource_select_fallback();
12887eaeb343SThomas Gleixner if (curr_clocksource == cs)
12897eaeb343SThomas Gleixner return -EBUSY;
12907eaeb343SThomas Gleixner }
12912aae7bcfSPeter Zijlstra
129239232ed5SBaolin Wang if (clocksource_is_suspend(cs)) {
129339232ed5SBaolin Wang /*
129439232ed5SBaolin Wang * Select and try to install a replacement suspend clocksource.
129539232ed5SBaolin Wang * If no replacement suspend clocksource, we will just let the
129639232ed5SBaolin Wang * clocksource go and have no suspend clocksource.
129739232ed5SBaolin Wang */
129839232ed5SBaolin Wang clocksource_suspend_select(true);
129939232ed5SBaolin Wang }
130039232ed5SBaolin Wang
13012aae7bcfSPeter Zijlstra clocksource_watchdog_lock(&flags);
13027eaeb343SThomas Gleixner clocksource_dequeue_watchdog(cs);
13037eaeb343SThomas Gleixner list_del_init(&cs->list);
13042aae7bcfSPeter Zijlstra clocksource_watchdog_unlock(&flags);
13052aae7bcfSPeter Zijlstra
13067eaeb343SThomas Gleixner return 0;
13077eaeb343SThomas Gleixner }
13087eaeb343SThomas Gleixner
13094713e22cSThomas Gleixner /**
13104713e22cSThomas Gleixner * clocksource_unregister - remove a registered clocksource
1311b1b73d09SKusanagi Kouichi * @cs: clocksource to be unregistered
13124713e22cSThomas Gleixner */
clocksource_unregister(struct clocksource * cs)1313a89c7edbSThomas Gleixner int clocksource_unregister(struct clocksource *cs)
13144713e22cSThomas Gleixner {
1315a89c7edbSThomas Gleixner int ret = 0;
1316a89c7edbSThomas Gleixner
131775c5158fSMartin Schwidefsky mutex_lock(&clocksource_mutex);
1318a89c7edbSThomas Gleixner if (!list_empty(&cs->list))
1319a89c7edbSThomas Gleixner ret = clocksource_unbind(cs);
132075c5158fSMartin Schwidefsky mutex_unlock(&clocksource_mutex);
1321a89c7edbSThomas Gleixner return ret;
13224713e22cSThomas Gleixner }
1323fb63a0ebSMartin Schwidefsky EXPORT_SYMBOL(clocksource_unregister);
13244713e22cSThomas Gleixner
13252b013700SDaniel Walker #ifdef CONFIG_SYSFS
1326734efb46Sjohn stultz /**
1327e87821d1SBaolin Wang * current_clocksource_show - sysfs interface for current clocksource
1328734efb46Sjohn stultz * @dev: unused
1329b1b73d09SKusanagi Kouichi * @attr: unused
1330734efb46Sjohn stultz * @buf: char buffer to be filled with clocksource list
1331734efb46Sjohn stultz *
1332734efb46Sjohn stultz * Provides sysfs interface for listing current clocksource.
1333734efb46Sjohn stultz */
current_clocksource_show(struct device * dev,struct device_attribute * attr,char * buf)1334e87821d1SBaolin Wang static ssize_t current_clocksource_show(struct device *dev,
1335e87821d1SBaolin Wang struct device_attribute *attr,
1336e87821d1SBaolin Wang char *buf)
1337734efb46Sjohn stultz {
13385e2cb101SMiao Xie ssize_t count = 0;
1339734efb46Sjohn stultz
134075c5158fSMartin Schwidefsky mutex_lock(&clocksource_mutex);
13415e2cb101SMiao Xie count = snprintf(buf, PAGE_SIZE, "%s\n", curr_clocksource->name);
134275c5158fSMartin Schwidefsky mutex_unlock(&clocksource_mutex);
1343734efb46Sjohn stultz
13445e2cb101SMiao Xie return count;
1345734efb46Sjohn stultz }
1346734efb46Sjohn stultz
sysfs_get_uname(const char * buf,char * dst,size_t cnt)1347891292a7SPatrick Palka ssize_t sysfs_get_uname(const char *buf, char *dst, size_t cnt)
134829b54078SThomas Gleixner {
134929b54078SThomas Gleixner size_t ret = cnt;
135029b54078SThomas Gleixner
135129b54078SThomas Gleixner /* strings from sysfs write are not 0 terminated! */
135229b54078SThomas Gleixner if (!cnt || cnt >= CS_NAME_LEN)
135329b54078SThomas Gleixner return -EINVAL;
135429b54078SThomas Gleixner
135529b54078SThomas Gleixner /* strip of \n: */
135629b54078SThomas Gleixner if (buf[cnt-1] == '\n')
135729b54078SThomas Gleixner cnt--;
135829b54078SThomas Gleixner if (cnt > 0)
135929b54078SThomas Gleixner memcpy(dst, buf, cnt);
136029b54078SThomas Gleixner dst[cnt] = 0;
136129b54078SThomas Gleixner return ret;
136229b54078SThomas Gleixner }
136329b54078SThomas Gleixner
1364734efb46Sjohn stultz /**
1365e87821d1SBaolin Wang * current_clocksource_store - interface for manually overriding clocksource
1366734efb46Sjohn stultz * @dev: unused
1367b1b73d09SKusanagi Kouichi * @attr: unused
1368734efb46Sjohn stultz * @buf: name of override clocksource
1369734efb46Sjohn stultz * @count: length of buffer
1370734efb46Sjohn stultz *
1371734efb46Sjohn stultz * Takes input from sysfs interface for manually overriding the default
1372b71a8eb0SUwe Kleine-König * clocksource selection.
1373734efb46Sjohn stultz */
current_clocksource_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1374e87821d1SBaolin Wang static ssize_t current_clocksource_store(struct device *dev,
1375d369a5d8SKay Sievers struct device_attribute *attr,
1376734efb46Sjohn stultz const char *buf, size_t count)
1377734efb46Sjohn stultz {
1378233bcb41SElad Wexler ssize_t ret;
1379734efb46Sjohn stultz
138075c5158fSMartin Schwidefsky mutex_lock(&clocksource_mutex);
1381734efb46Sjohn stultz
138203e13cf5SThomas Gleixner ret = sysfs_get_uname(buf, override_name, count);
138329b54078SThomas Gleixner if (ret >= 0)
1384f1b82746SMartin Schwidefsky clocksource_select();
1385734efb46Sjohn stultz
138675c5158fSMartin Schwidefsky mutex_unlock(&clocksource_mutex);
1387734efb46Sjohn stultz
1388734efb46Sjohn stultz return ret;
1389734efb46Sjohn stultz }
1390e87821d1SBaolin Wang static DEVICE_ATTR_RW(current_clocksource);
1391734efb46Sjohn stultz
1392734efb46Sjohn stultz /**
1393e87821d1SBaolin Wang * unbind_clocksource_store - interface for manually unbinding clocksource
13947eaeb343SThomas Gleixner * @dev: unused
13957eaeb343SThomas Gleixner * @attr: unused
13967eaeb343SThomas Gleixner * @buf: unused
13977eaeb343SThomas Gleixner * @count: length of buffer
13987eaeb343SThomas Gleixner *
13997eaeb343SThomas Gleixner * Takes input from sysfs interface for manually unbinding a clocksource.
14007eaeb343SThomas Gleixner */
unbind_clocksource_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1401e87821d1SBaolin Wang static ssize_t unbind_clocksource_store(struct device *dev,
14027eaeb343SThomas Gleixner struct device_attribute *attr,
14037eaeb343SThomas Gleixner const char *buf, size_t count)
14047eaeb343SThomas Gleixner {
14057eaeb343SThomas Gleixner struct clocksource *cs;
14067eaeb343SThomas Gleixner char name[CS_NAME_LEN];
1407233bcb41SElad Wexler ssize_t ret;
14087eaeb343SThomas Gleixner
140903e13cf5SThomas Gleixner ret = sysfs_get_uname(buf, name, count);
14107eaeb343SThomas Gleixner if (ret < 0)
14117eaeb343SThomas Gleixner return ret;
14127eaeb343SThomas Gleixner
14137eaeb343SThomas Gleixner ret = -ENODEV;
14147eaeb343SThomas Gleixner mutex_lock(&clocksource_mutex);
14157eaeb343SThomas Gleixner list_for_each_entry(cs, &clocksource_list, list) {
14167eaeb343SThomas Gleixner if (strcmp(cs->name, name))
14177eaeb343SThomas Gleixner continue;
14187eaeb343SThomas Gleixner ret = clocksource_unbind(cs);
14197eaeb343SThomas Gleixner break;
14207eaeb343SThomas Gleixner }
14217eaeb343SThomas Gleixner mutex_unlock(&clocksource_mutex);
14227eaeb343SThomas Gleixner
14237eaeb343SThomas Gleixner return ret ? ret : count;
14247eaeb343SThomas Gleixner }
1425e87821d1SBaolin Wang static DEVICE_ATTR_WO(unbind_clocksource);
14267eaeb343SThomas Gleixner
14277eaeb343SThomas Gleixner /**
1428e87821d1SBaolin Wang * available_clocksource_show - sysfs interface for listing clocksource
1429734efb46Sjohn stultz * @dev: unused
1430b1b73d09SKusanagi Kouichi * @attr: unused
1431734efb46Sjohn stultz * @buf: char buffer to be filled with clocksource list
1432734efb46Sjohn stultz *
1433734efb46Sjohn stultz * Provides sysfs interface for listing registered clocksources
1434734efb46Sjohn stultz */
available_clocksource_show(struct device * dev,struct device_attribute * attr,char * buf)1435e87821d1SBaolin Wang static ssize_t available_clocksource_show(struct device *dev,
1436d369a5d8SKay Sievers struct device_attribute *attr,
14374a0b2b4dSAndi Kleen char *buf)
1438734efb46Sjohn stultz {
14392e197586SMatthias Kaehlcke struct clocksource *src;
14405e2cb101SMiao Xie ssize_t count = 0;
1441734efb46Sjohn stultz
144275c5158fSMartin Schwidefsky mutex_lock(&clocksource_mutex);
14432e197586SMatthias Kaehlcke list_for_each_entry(src, &clocksource_list, list) {
1444cd6d95d8SThomas Gleixner /*
1445cd6d95d8SThomas Gleixner * Don't show non-HRES clocksource if the tick code is
1446cd6d95d8SThomas Gleixner * in one shot mode (highres=on or nohz=on)
1447cd6d95d8SThomas Gleixner */
1448cd6d95d8SThomas Gleixner if (!tick_oneshot_mode_active() ||
14493f68535aSjohn stultz (src->flags & CLOCK_SOURCE_VALID_FOR_HRES))
14505e2cb101SMiao Xie count += snprintf(buf + count,
14515e2cb101SMiao Xie max((ssize_t)PAGE_SIZE - count, (ssize_t)0),
14525e2cb101SMiao Xie "%s ", src->name);
1453734efb46Sjohn stultz }
145475c5158fSMartin Schwidefsky mutex_unlock(&clocksource_mutex);
1455734efb46Sjohn stultz
14565e2cb101SMiao Xie count += snprintf(buf + count,
14575e2cb101SMiao Xie max((ssize_t)PAGE_SIZE - count, (ssize_t)0), "\n");
1458734efb46Sjohn stultz
14595e2cb101SMiao Xie return count;
1460734efb46Sjohn stultz }
1461e87821d1SBaolin Wang static DEVICE_ATTR_RO(available_clocksource);
1462734efb46Sjohn stultz
146327263e8dSBaolin Wang static struct attribute *clocksource_attrs[] = {
146427263e8dSBaolin Wang &dev_attr_current_clocksource.attr,
146527263e8dSBaolin Wang &dev_attr_unbind_clocksource.attr,
146627263e8dSBaolin Wang &dev_attr_available_clocksource.attr,
146727263e8dSBaolin Wang NULL
146827263e8dSBaolin Wang };
146927263e8dSBaolin Wang ATTRIBUTE_GROUPS(clocksource);
147027263e8dSBaolin Wang
1471d369a5d8SKay Sievers static struct bus_type clocksource_subsys = {
1472af5ca3f4SKay Sievers .name = "clocksource",
1473d369a5d8SKay Sievers .dev_name = "clocksource",
1474734efb46Sjohn stultz };
1475734efb46Sjohn stultz
1476d369a5d8SKay Sievers static struct device device_clocksource = {
1477734efb46Sjohn stultz .id = 0,
1478d369a5d8SKay Sievers .bus = &clocksource_subsys,
147927263e8dSBaolin Wang .groups = clocksource_groups,
1480734efb46Sjohn stultz };
1481734efb46Sjohn stultz
init_clocksource_sysfs(void)1482ad596171Sjohn stultz static int __init init_clocksource_sysfs(void)
1483734efb46Sjohn stultz {
1484d369a5d8SKay Sievers int error = subsys_system_register(&clocksource_subsys, NULL);
1485734efb46Sjohn stultz
1486734efb46Sjohn stultz if (!error)
1487d369a5d8SKay Sievers error = device_register(&device_clocksource);
148827263e8dSBaolin Wang
1489734efb46Sjohn stultz return error;
1490734efb46Sjohn stultz }
1491734efb46Sjohn stultz
1492734efb46Sjohn stultz device_initcall(init_clocksource_sysfs);
14932b013700SDaniel Walker #endif /* CONFIG_SYSFS */
1494734efb46Sjohn stultz
1495734efb46Sjohn stultz /**
1496734efb46Sjohn stultz * boot_override_clocksource - boot clock override
1497734efb46Sjohn stultz * @str: override name
1498734efb46Sjohn stultz *
1499734efb46Sjohn stultz * Takes a clocksource= boot argument and uses it
1500734efb46Sjohn stultz * as the clocksource override name.
1501734efb46Sjohn stultz */
boot_override_clocksource(char * str)1502734efb46Sjohn stultz static int __init boot_override_clocksource(char* str)
1503734efb46Sjohn stultz {
150475c5158fSMartin Schwidefsky mutex_lock(&clocksource_mutex);
1505734efb46Sjohn stultz if (str)
150676edc27eSAzeem Shaikh strscpy(override_name, str, sizeof(override_name));
150775c5158fSMartin Schwidefsky mutex_unlock(&clocksource_mutex);
1508734efb46Sjohn stultz return 1;
1509734efb46Sjohn stultz }
1510734efb46Sjohn stultz
1511734efb46Sjohn stultz __setup("clocksource=", boot_override_clocksource);
1512734efb46Sjohn stultz
1513734efb46Sjohn stultz /**
1514734efb46Sjohn stultz * boot_override_clock - Compatibility layer for deprecated boot option
1515734efb46Sjohn stultz * @str: override name
1516734efb46Sjohn stultz *
1517734efb46Sjohn stultz * DEPRECATED! Takes a clock= boot argument and uses it
1518734efb46Sjohn stultz * as the clocksource override name
1519734efb46Sjohn stultz */
boot_override_clock(char * str)1520734efb46Sjohn stultz static int __init boot_override_clock(char* str)
1521734efb46Sjohn stultz {
15225d0cf410Sjohn stultz if (!strcmp(str, "pmtmr")) {
152345bbfe64SJoe Perches pr_warn("clock=pmtmr is deprecated - use clocksource=acpi_pm\n");
15245d0cf410Sjohn stultz return boot_override_clocksource("acpi_pm");
15255d0cf410Sjohn stultz }
152645bbfe64SJoe Perches pr_warn("clock= boot option is deprecated - use clocksource=xyz\n");
1527734efb46Sjohn stultz return boot_override_clocksource(str);
1528734efb46Sjohn stultz }
1529734efb46Sjohn stultz
1530734efb46Sjohn stultz __setup("clock=", boot_override_clock);
1531