xref: /openbmc/linux/kernel/time/timekeeping.c (revision 36fe4655)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  Kernel timekeeping code and accessor functions. Based on code from
4  *  timer.c, moved in commit 8524070b7982.
5  */
6 #include <linux/timekeeper_internal.h>
7 #include <linux/module.h>
8 #include <linux/interrupt.h>
9 #include <linux/percpu.h>
10 #include <linux/init.h>
11 #include <linux/mm.h>
12 #include <linux/nmi.h>
13 #include <linux/sched.h>
14 #include <linux/sched/loadavg.h>
15 #include <linux/sched/clock.h>
16 #include <linux/syscore_ops.h>
17 #include <linux/clocksource.h>
18 #include <linux/jiffies.h>
19 #include <linux/time.h>
20 #include <linux/tick.h>
21 #include <linux/stop_machine.h>
22 #include <linux/pvclock_gtod.h>
23 #include <linux/compiler.h>
24 #include <linux/audit.h>
25 
26 #include "tick-internal.h"
27 #include "ntp_internal.h"
28 #include "timekeeping_internal.h"
29 
30 #define TK_CLEAR_NTP		(1 << 0)
31 #define TK_MIRROR		(1 << 1)
32 #define TK_CLOCK_WAS_SET	(1 << 2)
33 
34 enum timekeeping_adv_mode {
35 	/* Update timekeeper when a tick has passed */
36 	TK_ADV_TICK,
37 
38 	/* Update timekeeper on a direct frequency change */
39 	TK_ADV_FREQ
40 };
41 
42 DEFINE_RAW_SPINLOCK(timekeeper_lock);
43 
44 /*
45  * The most important data for readout fits into a single 64 byte
46  * cache line.
47  */
48 static struct {
49 	seqcount_raw_spinlock_t	seq;
50 	struct timekeeper	timekeeper;
51 } tk_core ____cacheline_aligned = {
52 	.seq = SEQCNT_RAW_SPINLOCK_ZERO(tk_core.seq, &timekeeper_lock),
53 };
54 
55 static struct timekeeper shadow_timekeeper;
56 
57 /* flag for if timekeeping is suspended */
58 int __read_mostly timekeeping_suspended;
59 
60 /**
61  * struct tk_fast - NMI safe timekeeper
62  * @seq:	Sequence counter for protecting updates. The lowest bit
63  *		is the index for the tk_read_base array
64  * @base:	tk_read_base array. Access is indexed by the lowest bit of
65  *		@seq.
66  *
67  * See @update_fast_timekeeper() below.
68  */
69 struct tk_fast {
70 	seqcount_latch_t	seq;
71 	struct tk_read_base	base[2];
72 };
73 
74 /* Suspend-time cycles value for halted fast timekeeper. */
75 static u64 cycles_at_suspend;
76 
77 static u64 dummy_clock_read(struct clocksource *cs)
78 {
79 	if (timekeeping_suspended)
80 		return cycles_at_suspend;
81 	return local_clock();
82 }
83 
84 static struct clocksource dummy_clock = {
85 	.read = dummy_clock_read,
86 };
87 
88 /*
89  * Boot time initialization which allows local_clock() to be utilized
90  * during early boot when clocksources are not available. local_clock()
91  * returns nanoseconds already so no conversion is required, hence mult=1
92  * and shift=0. When the first proper clocksource is installed then
93  * the fast time keepers are updated with the correct values.
94  */
95 #define FAST_TK_INIT						\
96 	{							\
97 		.clock		= &dummy_clock,			\
98 		.mask		= CLOCKSOURCE_MASK(64),		\
99 		.mult		= 1,				\
100 		.shift		= 0,				\
101 	}
102 
103 static struct tk_fast tk_fast_mono ____cacheline_aligned = {
104 	.seq     = SEQCNT_LATCH_ZERO(tk_fast_mono.seq),
105 	.base[0] = FAST_TK_INIT,
106 	.base[1] = FAST_TK_INIT,
107 };
108 
109 static struct tk_fast tk_fast_raw  ____cacheline_aligned = {
110 	.seq     = SEQCNT_LATCH_ZERO(tk_fast_raw.seq),
111 	.base[0] = FAST_TK_INIT,
112 	.base[1] = FAST_TK_INIT,
113 };
114 
115 static inline void tk_normalize_xtime(struct timekeeper *tk)
116 {
117 	while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {
118 		tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
119 		tk->xtime_sec++;
120 	}
121 	while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) {
122 		tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
123 		tk->raw_sec++;
124 	}
125 }
126 
127 static inline struct timespec64 tk_xtime(const struct timekeeper *tk)
128 {
129 	struct timespec64 ts;
130 
131 	ts.tv_sec = tk->xtime_sec;
132 	ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
133 	return ts;
134 }
135 
136 static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts)
137 {
138 	tk->xtime_sec = ts->tv_sec;
139 	tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift;
140 }
141 
142 static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts)
143 {
144 	tk->xtime_sec += ts->tv_sec;
145 	tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift;
146 	tk_normalize_xtime(tk);
147 }
148 
149 static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm)
150 {
151 	struct timespec64 tmp;
152 
153 	/*
154 	 * Verify consistency of: offset_real = -wall_to_monotonic
155 	 * before modifying anything
156 	 */
157 	set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec,
158 					-tk->wall_to_monotonic.tv_nsec);
159 	WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp));
160 	tk->wall_to_monotonic = wtm;
161 	set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
162 	tk->offs_real = timespec64_to_ktime(tmp);
163 	tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0));
164 }
165 
166 static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
167 {
168 	tk->offs_boot = ktime_add(tk->offs_boot, delta);
169 	/*
170 	 * Timespec representation for VDSO update to avoid 64bit division
171 	 * on every update.
172 	 */
173 	tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot);
174 }
175 
176 /*
177  * tk_clock_read - atomic clocksource read() helper
178  *
179  * This helper is necessary to use in the read paths because, while the
180  * seqcount ensures we don't return a bad value while structures are updated,
181  * it doesn't protect from potential crashes. There is the possibility that
182  * the tkr's clocksource may change between the read reference, and the
183  * clock reference passed to the read function.  This can cause crashes if
184  * the wrong clocksource is passed to the wrong read function.
185  * This isn't necessary to use when holding the timekeeper_lock or doing
186  * a read of the fast-timekeeper tkrs (which is protected by its own locking
187  * and update logic).
188  */
189 static inline u64 tk_clock_read(const struct tk_read_base *tkr)
190 {
191 	struct clocksource *clock = READ_ONCE(tkr->clock);
192 
193 	return clock->read(clock);
194 }
195 
196 #ifdef CONFIG_DEBUG_TIMEKEEPING
197 #define WARNING_FREQ (HZ*300) /* 5 minute rate-limiting */
198 
199 static void timekeeping_check_update(struct timekeeper *tk, u64 offset)
200 {
201 
202 	u64 max_cycles = tk->tkr_mono.clock->max_cycles;
203 	const char *name = tk->tkr_mono.clock->name;
204 
205 	if (offset > max_cycles) {
206 		printk_deferred("WARNING: timekeeping: Cycle offset (%lld) is larger than allowed by the '%s' clock's max_cycles value (%lld): time overflow danger\n",
207 				offset, name, max_cycles);
208 		printk_deferred("         timekeeping: Your kernel is sick, but tries to cope by capping time updates\n");
209 	} else {
210 		if (offset > (max_cycles >> 1)) {
211 			printk_deferred("INFO: timekeeping: Cycle offset (%lld) is larger than the '%s' clock's 50%% safety margin (%lld)\n",
212 					offset, name, max_cycles >> 1);
213 			printk_deferred("      timekeeping: Your kernel is still fine, but is feeling a bit nervous\n");
214 		}
215 	}
216 
217 	if (tk->underflow_seen) {
218 		if (jiffies - tk->last_warning > WARNING_FREQ) {
219 			printk_deferred("WARNING: Underflow in clocksource '%s' observed, time update ignored.\n", name);
220 			printk_deferred("         Please report this, consider using a different clocksource, if possible.\n");
221 			printk_deferred("         Your kernel is probably still fine.\n");
222 			tk->last_warning = jiffies;
223 		}
224 		tk->underflow_seen = 0;
225 	}
226 
227 	if (tk->overflow_seen) {
228 		if (jiffies - tk->last_warning > WARNING_FREQ) {
229 			printk_deferred("WARNING: Overflow in clocksource '%s' observed, time update capped.\n", name);
230 			printk_deferred("         Please report this, consider using a different clocksource, if possible.\n");
231 			printk_deferred("         Your kernel is probably still fine.\n");
232 			tk->last_warning = jiffies;
233 		}
234 		tk->overflow_seen = 0;
235 	}
236 }
237 
238 static inline u64 timekeeping_get_delta(const struct tk_read_base *tkr)
239 {
240 	struct timekeeper *tk = &tk_core.timekeeper;
241 	u64 now, last, mask, max, delta;
242 	unsigned int seq;
243 
244 	/*
245 	 * Since we're called holding a seqcount, the data may shift
246 	 * under us while we're doing the calculation. This can cause
247 	 * false positives, since we'd note a problem but throw the
248 	 * results away. So nest another seqcount here to atomically
249 	 * grab the points we are checking with.
250 	 */
251 	do {
252 		seq = read_seqcount_begin(&tk_core.seq);
253 		now = tk_clock_read(tkr);
254 		last = tkr->cycle_last;
255 		mask = tkr->mask;
256 		max = tkr->clock->max_cycles;
257 	} while (read_seqcount_retry(&tk_core.seq, seq));
258 
259 	delta = clocksource_delta(now, last, mask);
260 
261 	/*
262 	 * Try to catch underflows by checking if we are seeing small
263 	 * mask-relative negative values.
264 	 */
265 	if (unlikely((~delta & mask) < (mask >> 3))) {
266 		tk->underflow_seen = 1;
267 		delta = 0;
268 	}
269 
270 	/* Cap delta value to the max_cycles values to avoid mult overflows */
271 	if (unlikely(delta > max)) {
272 		tk->overflow_seen = 1;
273 		delta = tkr->clock->max_cycles;
274 	}
275 
276 	return delta;
277 }
278 #else
279 static inline void timekeeping_check_update(struct timekeeper *tk, u64 offset)
280 {
281 }
282 static inline u64 timekeeping_get_delta(const struct tk_read_base *tkr)
283 {
284 	u64 cycle_now, delta;
285 
286 	/* read clocksource */
287 	cycle_now = tk_clock_read(tkr);
288 
289 	/* calculate the delta since the last update_wall_time */
290 	delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask);
291 
292 	return delta;
293 }
294 #endif
295 
296 /**
297  * tk_setup_internals - Set up internals to use clocksource clock.
298  *
299  * @tk:		The target timekeeper to setup.
300  * @clock:		Pointer to clocksource.
301  *
302  * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment
303  * pair and interval request.
304  *
305  * Unless you're the timekeeping code, you should not be using this!
306  */
307 static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
308 {
309 	u64 interval;
310 	u64 tmp, ntpinterval;
311 	struct clocksource *old_clock;
312 
313 	++tk->cs_was_changed_seq;
314 	old_clock = tk->tkr_mono.clock;
315 	tk->tkr_mono.clock = clock;
316 	tk->tkr_mono.mask = clock->mask;
317 	tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono);
318 
319 	tk->tkr_raw.clock = clock;
320 	tk->tkr_raw.mask = clock->mask;
321 	tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;
322 
323 	/* Do the ns -> cycle conversion first, using original mult */
324 	tmp = NTP_INTERVAL_LENGTH;
325 	tmp <<= clock->shift;
326 	ntpinterval = tmp;
327 	tmp += clock->mult/2;
328 	do_div(tmp, clock->mult);
329 	if (tmp == 0)
330 		tmp = 1;
331 
332 	interval = (u64) tmp;
333 	tk->cycle_interval = interval;
334 
335 	/* Go back from cycles -> shifted ns */
336 	tk->xtime_interval = interval * clock->mult;
337 	tk->xtime_remainder = ntpinterval - tk->xtime_interval;
338 	tk->raw_interval = interval * clock->mult;
339 
340 	 /* if changing clocks, convert xtime_nsec shift units */
341 	if (old_clock) {
342 		int shift_change = clock->shift - old_clock->shift;
343 		if (shift_change < 0) {
344 			tk->tkr_mono.xtime_nsec >>= -shift_change;
345 			tk->tkr_raw.xtime_nsec >>= -shift_change;
346 		} else {
347 			tk->tkr_mono.xtime_nsec <<= shift_change;
348 			tk->tkr_raw.xtime_nsec <<= shift_change;
349 		}
350 	}
351 
352 	tk->tkr_mono.shift = clock->shift;
353 	tk->tkr_raw.shift = clock->shift;
354 
355 	tk->ntp_error = 0;
356 	tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
357 	tk->ntp_tick = ntpinterval << tk->ntp_error_shift;
358 
359 	/*
360 	 * The timekeeper keeps its own mult values for the currently
361 	 * active clocksource. These value will be adjusted via NTP
362 	 * to counteract clock drifting.
363 	 */
364 	tk->tkr_mono.mult = clock->mult;
365 	tk->tkr_raw.mult = clock->mult;
366 	tk->ntp_err_mult = 0;
367 	tk->skip_second_overflow = 0;
368 }
369 
370 /* Timekeeper helper functions. */
371 
372 static inline u64 timekeeping_delta_to_ns(const struct tk_read_base *tkr, u64 delta)
373 {
374 	u64 nsec;
375 
376 	nsec = delta * tkr->mult + tkr->xtime_nsec;
377 	nsec >>= tkr->shift;
378 
379 	return nsec;
380 }
381 
382 static inline u64 timekeeping_get_ns(const struct tk_read_base *tkr)
383 {
384 	u64 delta;
385 
386 	delta = timekeeping_get_delta(tkr);
387 	return timekeeping_delta_to_ns(tkr, delta);
388 }
389 
390 static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles)
391 {
392 	u64 delta;
393 
394 	/* calculate the delta since the last update_wall_time */
395 	delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask);
396 	return timekeeping_delta_to_ns(tkr, delta);
397 }
398 
399 /**
400  * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper.
401  * @tkr: Timekeeping readout base from which we take the update
402  * @tkf: Pointer to NMI safe timekeeper
403  *
404  * We want to use this from any context including NMI and tracing /
405  * instrumenting the timekeeping code itself.
406  *
407  * Employ the latch technique; see @raw_write_seqcount_latch.
408  *
409  * So if a NMI hits the update of base[0] then it will use base[1]
410  * which is still consistent. In the worst case this can result is a
411  * slightly wrong timestamp (a few nanoseconds). See
412  * @ktime_get_mono_fast_ns.
413  */
414 static void update_fast_timekeeper(const struct tk_read_base *tkr,
415 				   struct tk_fast *tkf)
416 {
417 	struct tk_read_base *base = tkf->base;
418 
419 	/* Force readers off to base[1] */
420 	raw_write_seqcount_latch(&tkf->seq);
421 
422 	/* Update base[0] */
423 	memcpy(base, tkr, sizeof(*base));
424 
425 	/* Force readers back to base[0] */
426 	raw_write_seqcount_latch(&tkf->seq);
427 
428 	/* Update base[1] */
429 	memcpy(base + 1, base, sizeof(*base));
430 }
431 
432 static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
433 {
434 	struct tk_read_base *tkr;
435 	unsigned int seq;
436 	u64 now;
437 
438 	do {
439 		seq = raw_read_seqcount_latch(&tkf->seq);
440 		tkr = tkf->base + (seq & 0x01);
441 		now = ktime_to_ns(tkr->base);
442 
443 		now += timekeeping_delta_to_ns(tkr,
444 				clocksource_delta(
445 					tk_clock_read(tkr),
446 					tkr->cycle_last,
447 					tkr->mask));
448 	} while (read_seqcount_latch_retry(&tkf->seq, seq));
449 
450 	return now;
451 }
452 
453 /**
454  * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic
455  *
456  * This timestamp is not guaranteed to be monotonic across an update.
457  * The timestamp is calculated by:
458  *
459  *	now = base_mono + clock_delta * slope
460  *
461  * So if the update lowers the slope, readers who are forced to the
462  * not yet updated second array are still using the old steeper slope.
463  *
464  * tmono
465  * ^
466  * |    o  n
467  * |   o n
468  * |  u
469  * | o
470  * |o
471  * |12345678---> reader order
472  *
473  * o = old slope
474  * u = update
475  * n = new slope
476  *
477  * So reader 6 will observe time going backwards versus reader 5.
478  *
479  * While other CPUs are likely to be able to observe that, the only way
480  * for a CPU local observation is when an NMI hits in the middle of
481  * the update. Timestamps taken from that NMI context might be ahead
482  * of the following timestamps. Callers need to be aware of that and
483  * deal with it.
484  */
485 u64 ktime_get_mono_fast_ns(void)
486 {
487 	return __ktime_get_fast_ns(&tk_fast_mono);
488 }
489 EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns);
490 
491 /**
492  * ktime_get_raw_fast_ns - Fast NMI safe access to clock monotonic raw
493  *
494  * Contrary to ktime_get_mono_fast_ns() this is always correct because the
495  * conversion factor is not affected by NTP/PTP correction.
496  */
497 u64 ktime_get_raw_fast_ns(void)
498 {
499 	return __ktime_get_fast_ns(&tk_fast_raw);
500 }
501 EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns);
502 
503 /**
504  * ktime_get_boot_fast_ns - NMI safe and fast access to boot clock.
505  *
506  * To keep it NMI safe since we're accessing from tracing, we're not using a
507  * separate timekeeper with updates to monotonic clock and boot offset
508  * protected with seqcounts. This has the following minor side effects:
509  *
510  * (1) Its possible that a timestamp be taken after the boot offset is updated
511  * but before the timekeeper is updated. If this happens, the new boot offset
512  * is added to the old timekeeping making the clock appear to update slightly
513  * earlier:
514  *    CPU 0                                        CPU 1
515  *    timekeeping_inject_sleeptime64()
516  *    __timekeeping_inject_sleeptime(tk, delta);
517  *                                                 timestamp();
518  *    timekeeping_update(tk, TK_CLEAR_NTP...);
519  *
520  * (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be
521  * partially updated.  Since the tk->offs_boot update is a rare event, this
522  * should be a rare occurrence which postprocessing should be able to handle.
523  *
524  * The caveats vs. timestamp ordering as documented for ktime_get_fast_ns()
525  * apply as well.
526  */
527 u64 notrace ktime_get_boot_fast_ns(void)
528 {
529 	struct timekeeper *tk = &tk_core.timekeeper;
530 
531 	return (ktime_get_mono_fast_ns() + ktime_to_ns(tk->offs_boot));
532 }
533 EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns);
534 
535 static __always_inline u64 __ktime_get_real_fast(struct tk_fast *tkf, u64 *mono)
536 {
537 	struct tk_read_base *tkr;
538 	u64 basem, baser, delta;
539 	unsigned int seq;
540 
541 	do {
542 		seq = raw_read_seqcount_latch(&tkf->seq);
543 		tkr = tkf->base + (seq & 0x01);
544 		basem = ktime_to_ns(tkr->base);
545 		baser = ktime_to_ns(tkr->base_real);
546 
547 		delta = timekeeping_delta_to_ns(tkr,
548 				clocksource_delta(tk_clock_read(tkr),
549 				tkr->cycle_last, tkr->mask));
550 	} while (read_seqcount_latch_retry(&tkf->seq, seq));
551 
552 	if (mono)
553 		*mono = basem + delta;
554 	return baser + delta;
555 }
556 
557 /**
558  * ktime_get_real_fast_ns: - NMI safe and fast access to clock realtime.
559  *
560  * See ktime_get_fast_ns() for documentation of the time stamp ordering.
561  */
562 u64 ktime_get_real_fast_ns(void)
563 {
564 	return __ktime_get_real_fast(&tk_fast_mono, NULL);
565 }
566 EXPORT_SYMBOL_GPL(ktime_get_real_fast_ns);
567 
568 /**
569  * ktime_get_fast_timestamps: - NMI safe timestamps
570  * @snapshot:	Pointer to timestamp storage
571  *
572  * Stores clock monotonic, boottime and realtime timestamps.
573  *
574  * Boot time is a racy access on 32bit systems if the sleep time injection
575  * happens late during resume and not in timekeeping_resume(). That could
576  * be avoided by expanding struct tk_read_base with boot offset for 32bit
577  * and adding more overhead to the update. As this is a hard to observe
578  * once per resume event which can be filtered with reasonable effort using
579  * the accurate mono/real timestamps, it's probably not worth the trouble.
580  *
581  * Aside of that it might be possible on 32 and 64 bit to observe the
582  * following when the sleep time injection happens late:
583  *
584  * CPU 0				CPU 1
585  * timekeeping_resume()
586  * ktime_get_fast_timestamps()
587  *	mono, real = __ktime_get_real_fast()
588  *					inject_sleep_time()
589  *					   update boot offset
590  *	boot = mono + bootoffset;
591  *
592  * That means that boot time already has the sleep time adjustment, but
593  * real time does not. On the next readout both are in sync again.
594  *
595  * Preventing this for 64bit is not really feasible without destroying the
596  * careful cache layout of the timekeeper because the sequence count and
597  * struct tk_read_base would then need two cache lines instead of one.
598  *
599  * Access to the time keeper clock source is disabled accross the innermost
600  * steps of suspend/resume. The accessors still work, but the timestamps
601  * are frozen until time keeping is resumed which happens very early.
602  *
603  * For regular suspend/resume there is no observable difference vs. sched
604  * clock, but it might affect some of the nasty low level debug printks.
605  *
606  * OTOH, access to sched clock is not guaranteed accross suspend/resume on
607  * all systems either so it depends on the hardware in use.
608  *
609  * If that turns out to be a real problem then this could be mitigated by
610  * using sched clock in a similar way as during early boot. But it's not as
611  * trivial as on early boot because it needs some careful protection
612  * against the clock monotonic timestamp jumping backwards on resume.
613  */
614 void ktime_get_fast_timestamps(struct ktime_timestamps *snapshot)
615 {
616 	struct timekeeper *tk = &tk_core.timekeeper;
617 
618 	snapshot->real = __ktime_get_real_fast(&tk_fast_mono, &snapshot->mono);
619 	snapshot->boot = snapshot->mono + ktime_to_ns(data_race(tk->offs_boot));
620 }
621 
622 /**
623  * halt_fast_timekeeper - Prevent fast timekeeper from accessing clocksource.
624  * @tk: Timekeeper to snapshot.
625  *
626  * It generally is unsafe to access the clocksource after timekeeping has been
627  * suspended, so take a snapshot of the readout base of @tk and use it as the
628  * fast timekeeper's readout base while suspended.  It will return the same
629  * number of cycles every time until timekeeping is resumed at which time the
630  * proper readout base for the fast timekeeper will be restored automatically.
631  */
632 static void halt_fast_timekeeper(const struct timekeeper *tk)
633 {
634 	static struct tk_read_base tkr_dummy;
635 	const struct tk_read_base *tkr = &tk->tkr_mono;
636 
637 	memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
638 	cycles_at_suspend = tk_clock_read(tkr);
639 	tkr_dummy.clock = &dummy_clock;
640 	tkr_dummy.base_real = tkr->base + tk->offs_real;
641 	update_fast_timekeeper(&tkr_dummy, &tk_fast_mono);
642 
643 	tkr = &tk->tkr_raw;
644 	memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
645 	tkr_dummy.clock = &dummy_clock;
646 	update_fast_timekeeper(&tkr_dummy, &tk_fast_raw);
647 }
648 
649 static RAW_NOTIFIER_HEAD(pvclock_gtod_chain);
650 
651 static void update_pvclock_gtod(struct timekeeper *tk, bool was_set)
652 {
653 	raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk);
654 }
655 
656 /**
657  * pvclock_gtod_register_notifier - register a pvclock timedata update listener
658  * @nb: Pointer to the notifier block to register
659  */
660 int pvclock_gtod_register_notifier(struct notifier_block *nb)
661 {
662 	struct timekeeper *tk = &tk_core.timekeeper;
663 	unsigned long flags;
664 	int ret;
665 
666 	raw_spin_lock_irqsave(&timekeeper_lock, flags);
667 	ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb);
668 	update_pvclock_gtod(tk, true);
669 	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
670 
671 	return ret;
672 }
673 EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier);
674 
675 /**
676  * pvclock_gtod_unregister_notifier - unregister a pvclock
677  * timedata update listener
678  * @nb: Pointer to the notifier block to unregister
679  */
680 int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
681 {
682 	unsigned long flags;
683 	int ret;
684 
685 	raw_spin_lock_irqsave(&timekeeper_lock, flags);
686 	ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb);
687 	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
688 
689 	return ret;
690 }
691 EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);
692 
693 /*
694  * tk_update_leap_state - helper to update the next_leap_ktime
695  */
696 static inline void tk_update_leap_state(struct timekeeper *tk)
697 {
698 	tk->next_leap_ktime = ntp_get_next_leap();
699 	if (tk->next_leap_ktime != KTIME_MAX)
700 		/* Convert to monotonic time */
701 		tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real);
702 }
703 
704 /*
705  * Update the ktime_t based scalar nsec members of the timekeeper
706  */
707 static inline void tk_update_ktime_data(struct timekeeper *tk)
708 {
709 	u64 seconds;
710 	u32 nsec;
711 
712 	/*
713 	 * The xtime based monotonic readout is:
714 	 *	nsec = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec + now();
715 	 * The ktime based monotonic readout is:
716 	 *	nsec = base_mono + now();
717 	 * ==> base_mono = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec
718 	 */
719 	seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec);
720 	nsec = (u32) tk->wall_to_monotonic.tv_nsec;
721 	tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec);
722 
723 	/*
724 	 * The sum of the nanoseconds portions of xtime and
725 	 * wall_to_monotonic can be greater/equal one second. Take
726 	 * this into account before updating tk->ktime_sec.
727 	 */
728 	nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
729 	if (nsec >= NSEC_PER_SEC)
730 		seconds++;
731 	tk->ktime_sec = seconds;
732 
733 	/* Update the monotonic raw base */
734 	tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC);
735 }
736 
737 /* must hold timekeeper_lock */
738 static void timekeeping_update(struct timekeeper *tk, unsigned int action)
739 {
740 	if (action & TK_CLEAR_NTP) {
741 		tk->ntp_error = 0;
742 		ntp_clear();
743 	}
744 
745 	tk_update_leap_state(tk);
746 	tk_update_ktime_data(tk);
747 
748 	update_vsyscall(tk);
749 	update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);
750 
751 	tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real;
752 	update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono);
753 	update_fast_timekeeper(&tk->tkr_raw,  &tk_fast_raw);
754 
755 	if (action & TK_CLOCK_WAS_SET)
756 		tk->clock_was_set_seq++;
757 	/*
758 	 * The mirroring of the data to the shadow-timekeeper needs
759 	 * to happen last here to ensure we don't over-write the
760 	 * timekeeper structure on the next update with stale data
761 	 */
762 	if (action & TK_MIRROR)
763 		memcpy(&shadow_timekeeper, &tk_core.timekeeper,
764 		       sizeof(tk_core.timekeeper));
765 }
766 
767 /**
768  * timekeeping_forward_now - update clock to the current time
769  * @tk:		Pointer to the timekeeper to update
770  *
771  * Forward the current clock to update its state since the last call to
772  * update_wall_time(). This is useful before significant clock changes,
773  * as it avoids having to deal with this time offset explicitly.
774  */
775 static void timekeeping_forward_now(struct timekeeper *tk)
776 {
777 	u64 cycle_now, delta;
778 
779 	cycle_now = tk_clock_read(&tk->tkr_mono);
780 	delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
781 	tk->tkr_mono.cycle_last = cycle_now;
782 	tk->tkr_raw.cycle_last  = cycle_now;
783 
784 	tk->tkr_mono.xtime_nsec += delta * tk->tkr_mono.mult;
785 	tk->tkr_raw.xtime_nsec += delta * tk->tkr_raw.mult;
786 
787 	tk_normalize_xtime(tk);
788 }
789 
790 /**
791  * ktime_get_real_ts64 - Returns the time of day in a timespec64.
792  * @ts:		pointer to the timespec to be set
793  *
794  * Returns the time of day in a timespec64 (WARN if suspended).
795  */
796 void ktime_get_real_ts64(struct timespec64 *ts)
797 {
798 	struct timekeeper *tk = &tk_core.timekeeper;
799 	unsigned int seq;
800 	u64 nsecs;
801 
802 	WARN_ON(timekeeping_suspended);
803 
804 	do {
805 		seq = read_seqcount_begin(&tk_core.seq);
806 
807 		ts->tv_sec = tk->xtime_sec;
808 		nsecs = timekeeping_get_ns(&tk->tkr_mono);
809 
810 	} while (read_seqcount_retry(&tk_core.seq, seq));
811 
812 	ts->tv_nsec = 0;
813 	timespec64_add_ns(ts, nsecs);
814 }
815 EXPORT_SYMBOL(ktime_get_real_ts64);
816 
817 ktime_t ktime_get(void)
818 {
819 	struct timekeeper *tk = &tk_core.timekeeper;
820 	unsigned int seq;
821 	ktime_t base;
822 	u64 nsecs;
823 
824 	WARN_ON(timekeeping_suspended);
825 
826 	do {
827 		seq = read_seqcount_begin(&tk_core.seq);
828 		base = tk->tkr_mono.base;
829 		nsecs = timekeeping_get_ns(&tk->tkr_mono);
830 
831 	} while (read_seqcount_retry(&tk_core.seq, seq));
832 
833 	return ktime_add_ns(base, nsecs);
834 }
835 EXPORT_SYMBOL_GPL(ktime_get);
836 
837 u32 ktime_get_resolution_ns(void)
838 {
839 	struct timekeeper *tk = &tk_core.timekeeper;
840 	unsigned int seq;
841 	u32 nsecs;
842 
843 	WARN_ON(timekeeping_suspended);
844 
845 	do {
846 		seq = read_seqcount_begin(&tk_core.seq);
847 		nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift;
848 	} while (read_seqcount_retry(&tk_core.seq, seq));
849 
850 	return nsecs;
851 }
852 EXPORT_SYMBOL_GPL(ktime_get_resolution_ns);
853 
854 static ktime_t *offsets[TK_OFFS_MAX] = {
855 	[TK_OFFS_REAL]	= &tk_core.timekeeper.offs_real,
856 	[TK_OFFS_BOOT]	= &tk_core.timekeeper.offs_boot,
857 	[TK_OFFS_TAI]	= &tk_core.timekeeper.offs_tai,
858 };
859 
860 ktime_t ktime_get_with_offset(enum tk_offsets offs)
861 {
862 	struct timekeeper *tk = &tk_core.timekeeper;
863 	unsigned int seq;
864 	ktime_t base, *offset = offsets[offs];
865 	u64 nsecs;
866 
867 	WARN_ON(timekeeping_suspended);
868 
869 	do {
870 		seq = read_seqcount_begin(&tk_core.seq);
871 		base = ktime_add(tk->tkr_mono.base, *offset);
872 		nsecs = timekeeping_get_ns(&tk->tkr_mono);
873 
874 	} while (read_seqcount_retry(&tk_core.seq, seq));
875 
876 	return ktime_add_ns(base, nsecs);
877 
878 }
879 EXPORT_SYMBOL_GPL(ktime_get_with_offset);
880 
881 ktime_t ktime_get_coarse_with_offset(enum tk_offsets offs)
882 {
883 	struct timekeeper *tk = &tk_core.timekeeper;
884 	unsigned int seq;
885 	ktime_t base, *offset = offsets[offs];
886 	u64 nsecs;
887 
888 	WARN_ON(timekeeping_suspended);
889 
890 	do {
891 		seq = read_seqcount_begin(&tk_core.seq);
892 		base = ktime_add(tk->tkr_mono.base, *offset);
893 		nsecs = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;
894 
895 	} while (read_seqcount_retry(&tk_core.seq, seq));
896 
897 	return ktime_add_ns(base, nsecs);
898 }
899 EXPORT_SYMBOL_GPL(ktime_get_coarse_with_offset);
900 
901 /**
902  * ktime_mono_to_any() - convert mononotic time to any other time
903  * @tmono:	time to convert.
904  * @offs:	which offset to use
905  */
906 ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs)
907 {
908 	ktime_t *offset = offsets[offs];
909 	unsigned int seq;
910 	ktime_t tconv;
911 
912 	do {
913 		seq = read_seqcount_begin(&tk_core.seq);
914 		tconv = ktime_add(tmono, *offset);
915 	} while (read_seqcount_retry(&tk_core.seq, seq));
916 
917 	return tconv;
918 }
919 EXPORT_SYMBOL_GPL(ktime_mono_to_any);
920 
921 /**
922  * ktime_get_raw - Returns the raw monotonic time in ktime_t format
923  */
924 ktime_t ktime_get_raw(void)
925 {
926 	struct timekeeper *tk = &tk_core.timekeeper;
927 	unsigned int seq;
928 	ktime_t base;
929 	u64 nsecs;
930 
931 	do {
932 		seq = read_seqcount_begin(&tk_core.seq);
933 		base = tk->tkr_raw.base;
934 		nsecs = timekeeping_get_ns(&tk->tkr_raw);
935 
936 	} while (read_seqcount_retry(&tk_core.seq, seq));
937 
938 	return ktime_add_ns(base, nsecs);
939 }
940 EXPORT_SYMBOL_GPL(ktime_get_raw);
941 
942 /**
943  * ktime_get_ts64 - get the monotonic clock in timespec64 format
944  * @ts:		pointer to timespec variable
945  *
946  * The function calculates the monotonic clock from the realtime
947  * clock and the wall_to_monotonic offset and stores the result
948  * in normalized timespec64 format in the variable pointed to by @ts.
949  */
950 void ktime_get_ts64(struct timespec64 *ts)
951 {
952 	struct timekeeper *tk = &tk_core.timekeeper;
953 	struct timespec64 tomono;
954 	unsigned int seq;
955 	u64 nsec;
956 
957 	WARN_ON(timekeeping_suspended);
958 
959 	do {
960 		seq = read_seqcount_begin(&tk_core.seq);
961 		ts->tv_sec = tk->xtime_sec;
962 		nsec = timekeeping_get_ns(&tk->tkr_mono);
963 		tomono = tk->wall_to_monotonic;
964 
965 	} while (read_seqcount_retry(&tk_core.seq, seq));
966 
967 	ts->tv_sec += tomono.tv_sec;
968 	ts->tv_nsec = 0;
969 	timespec64_add_ns(ts, nsec + tomono.tv_nsec);
970 }
971 EXPORT_SYMBOL_GPL(ktime_get_ts64);
972 
973 /**
974  * ktime_get_seconds - Get the seconds portion of CLOCK_MONOTONIC
975  *
976  * Returns the seconds portion of CLOCK_MONOTONIC with a single non
977  * serialized read. tk->ktime_sec is of type 'unsigned long' so this
978  * works on both 32 and 64 bit systems. On 32 bit systems the readout
979  * covers ~136 years of uptime which should be enough to prevent
980  * premature wrap arounds.
981  */
982 time64_t ktime_get_seconds(void)
983 {
984 	struct timekeeper *tk = &tk_core.timekeeper;
985 
986 	WARN_ON(timekeeping_suspended);
987 	return tk->ktime_sec;
988 }
989 EXPORT_SYMBOL_GPL(ktime_get_seconds);
990 
991 /**
992  * ktime_get_real_seconds - Get the seconds portion of CLOCK_REALTIME
993  *
994  * Returns the wall clock seconds since 1970. This replaces the
995  * get_seconds() interface which is not y2038 safe on 32bit systems.
996  *
997  * For 64bit systems the fast access to tk->xtime_sec is preserved. On
998  * 32bit systems the access must be protected with the sequence
999  * counter to provide "atomic" access to the 64bit tk->xtime_sec
1000  * value.
1001  */
1002 time64_t ktime_get_real_seconds(void)
1003 {
1004 	struct timekeeper *tk = &tk_core.timekeeper;
1005 	time64_t seconds;
1006 	unsigned int seq;
1007 
1008 	if (IS_ENABLED(CONFIG_64BIT))
1009 		return tk->xtime_sec;
1010 
1011 	do {
1012 		seq = read_seqcount_begin(&tk_core.seq);
1013 		seconds = tk->xtime_sec;
1014 
1015 	} while (read_seqcount_retry(&tk_core.seq, seq));
1016 
1017 	return seconds;
1018 }
1019 EXPORT_SYMBOL_GPL(ktime_get_real_seconds);
1020 
1021 /**
1022  * __ktime_get_real_seconds - The same as ktime_get_real_seconds
1023  * but without the sequence counter protect. This internal function
1024  * is called just when timekeeping lock is already held.
1025  */
1026 noinstr time64_t __ktime_get_real_seconds(void)
1027 {
1028 	struct timekeeper *tk = &tk_core.timekeeper;
1029 
1030 	return tk->xtime_sec;
1031 }
1032 
1033 /**
1034  * ktime_get_snapshot - snapshots the realtime/monotonic raw clocks with counter
1035  * @systime_snapshot:	pointer to struct receiving the system time snapshot
1036  */
1037 void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
1038 {
1039 	struct timekeeper *tk = &tk_core.timekeeper;
1040 	unsigned int seq;
1041 	ktime_t base_raw;
1042 	ktime_t base_real;
1043 	u64 nsec_raw;
1044 	u64 nsec_real;
1045 	u64 now;
1046 
1047 	WARN_ON_ONCE(timekeeping_suspended);
1048 
1049 	do {
1050 		seq = read_seqcount_begin(&tk_core.seq);
1051 		now = tk_clock_read(&tk->tkr_mono);
1052 		systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq;
1053 		systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;
1054 		base_real = ktime_add(tk->tkr_mono.base,
1055 				      tk_core.timekeeper.offs_real);
1056 		base_raw = tk->tkr_raw.base;
1057 		nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now);
1058 		nsec_raw  = timekeeping_cycles_to_ns(&tk->tkr_raw, now);
1059 	} while (read_seqcount_retry(&tk_core.seq, seq));
1060 
1061 	systime_snapshot->cycles = now;
1062 	systime_snapshot->real = ktime_add_ns(base_real, nsec_real);
1063 	systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw);
1064 }
1065 EXPORT_SYMBOL_GPL(ktime_get_snapshot);
1066 
1067 /* Scale base by mult/div checking for overflow */
1068 static int scale64_check_overflow(u64 mult, u64 div, u64 *base)
1069 {
1070 	u64 tmp, rem;
1071 
1072 	tmp = div64_u64_rem(*base, div, &rem);
1073 
1074 	if (((int)sizeof(u64)*8 - fls64(mult) < fls64(tmp)) ||
1075 	    ((int)sizeof(u64)*8 - fls64(mult) < fls64(rem)))
1076 		return -EOVERFLOW;
1077 	tmp *= mult;
1078 
1079 	rem = div64_u64(rem * mult, div);
1080 	*base = tmp + rem;
1081 	return 0;
1082 }
1083 
1084 /**
1085  * adjust_historical_crosststamp - adjust crosstimestamp previous to current interval
1086  * @history:			Snapshot representing start of history
1087  * @partial_history_cycles:	Cycle offset into history (fractional part)
1088  * @total_history_cycles:	Total history length in cycles
1089  * @discontinuity:		True indicates clock was set on history period
1090  * @ts:				Cross timestamp that should be adjusted using
1091  *	partial/total ratio
1092  *
1093  * Helper function used by get_device_system_crosststamp() to correct the
1094  * crosstimestamp corresponding to the start of the current interval to the
1095  * system counter value (timestamp point) provided by the driver. The
1096  * total_history_* quantities are the total history starting at the provided
1097  * reference point and ending at the start of the current interval. The cycle
1098  * count between the driver timestamp point and the start of the current
1099  * interval is partial_history_cycles.
1100  */
1101 static int adjust_historical_crosststamp(struct system_time_snapshot *history,
1102 					 u64 partial_history_cycles,
1103 					 u64 total_history_cycles,
1104 					 bool discontinuity,
1105 					 struct system_device_crosststamp *ts)
1106 {
1107 	struct timekeeper *tk = &tk_core.timekeeper;
1108 	u64 corr_raw, corr_real;
1109 	bool interp_forward;
1110 	int ret;
1111 
1112 	if (total_history_cycles == 0 || partial_history_cycles == 0)
1113 		return 0;
1114 
1115 	/* Interpolate shortest distance from beginning or end of history */
1116 	interp_forward = partial_history_cycles > total_history_cycles / 2;
1117 	partial_history_cycles = interp_forward ?
1118 		total_history_cycles - partial_history_cycles :
1119 		partial_history_cycles;
1120 
1121 	/*
1122 	 * Scale the monotonic raw time delta by:
1123 	 *	partial_history_cycles / total_history_cycles
1124 	 */
1125 	corr_raw = (u64)ktime_to_ns(
1126 		ktime_sub(ts->sys_monoraw, history->raw));
1127 	ret = scale64_check_overflow(partial_history_cycles,
1128 				     total_history_cycles, &corr_raw);
1129 	if (ret)
1130 		return ret;
1131 
1132 	/*
1133 	 * If there is a discontinuity in the history, scale monotonic raw
1134 	 *	correction by:
1135 	 *	mult(real)/mult(raw) yielding the realtime correction
1136 	 * Otherwise, calculate the realtime correction similar to monotonic
1137 	 *	raw calculation
1138 	 */
1139 	if (discontinuity) {
1140 		corr_real = mul_u64_u32_div
1141 			(corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult);
1142 	} else {
1143 		corr_real = (u64)ktime_to_ns(
1144 			ktime_sub(ts->sys_realtime, history->real));
1145 		ret = scale64_check_overflow(partial_history_cycles,
1146 					     total_history_cycles, &corr_real);
1147 		if (ret)
1148 			return ret;
1149 	}
1150 
1151 	/* Fixup monotonic raw and real time time values */
1152 	if (interp_forward) {
1153 		ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw);
1154 		ts->sys_realtime = ktime_add_ns(history->real, corr_real);
1155 	} else {
1156 		ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw);
1157 		ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real);
1158 	}
1159 
1160 	return 0;
1161 }
1162 
1163 /*
1164  * cycle_between - true if test occurs chronologically between before and after
1165  */
1166 static bool cycle_between(u64 before, u64 test, u64 after)
1167 {
1168 	if (test > before && test < after)
1169 		return true;
1170 	if (test < before && before > after)
1171 		return true;
1172 	return false;
1173 }
1174 
1175 /**
1176  * get_device_system_crosststamp - Synchronously capture system/device timestamp
1177  * @get_time_fn:	Callback to get simultaneous device time and
1178  *	system counter from the device driver
1179  * @ctx:		Context passed to get_time_fn()
1180  * @history_begin:	Historical reference point used to interpolate system
1181  *	time when counter provided by the driver is before the current interval
1182  * @xtstamp:		Receives simultaneously captured system and device time
1183  *
1184  * Reads a timestamp from a device and correlates it to system time
1185  */
1186 int get_device_system_crosststamp(int (*get_time_fn)
1187 				  (ktime_t *device_time,
1188 				   struct system_counterval_t *sys_counterval,
1189 				   void *ctx),
1190 				  void *ctx,
1191 				  struct system_time_snapshot *history_begin,
1192 				  struct system_device_crosststamp *xtstamp)
1193 {
1194 	struct system_counterval_t system_counterval;
1195 	struct timekeeper *tk = &tk_core.timekeeper;
1196 	u64 cycles, now, interval_start;
1197 	unsigned int clock_was_set_seq = 0;
1198 	ktime_t base_real, base_raw;
1199 	u64 nsec_real, nsec_raw;
1200 	u8 cs_was_changed_seq;
1201 	unsigned int seq;
1202 	bool do_interp;
1203 	int ret;
1204 
1205 	do {
1206 		seq = read_seqcount_begin(&tk_core.seq);
1207 		/*
1208 		 * Try to synchronously capture device time and a system
1209 		 * counter value calling back into the device driver
1210 		 */
1211 		ret = get_time_fn(&xtstamp->device, &system_counterval, ctx);
1212 		if (ret)
1213 			return ret;
1214 
1215 		/*
1216 		 * Verify that the clocksource associated with the captured
1217 		 * system counter value is the same as the currently installed
1218 		 * timekeeper clocksource
1219 		 */
1220 		if (tk->tkr_mono.clock != system_counterval.cs)
1221 			return -ENODEV;
1222 		cycles = system_counterval.cycles;
1223 
1224 		/*
1225 		 * Check whether the system counter value provided by the
1226 		 * device driver is on the current timekeeping interval.
1227 		 */
1228 		now = tk_clock_read(&tk->tkr_mono);
1229 		interval_start = tk->tkr_mono.cycle_last;
1230 		if (!cycle_between(interval_start, cycles, now)) {
1231 			clock_was_set_seq = tk->clock_was_set_seq;
1232 			cs_was_changed_seq = tk->cs_was_changed_seq;
1233 			cycles = interval_start;
1234 			do_interp = true;
1235 		} else {
1236 			do_interp = false;
1237 		}
1238 
1239 		base_real = ktime_add(tk->tkr_mono.base,
1240 				      tk_core.timekeeper.offs_real);
1241 		base_raw = tk->tkr_raw.base;
1242 
1243 		nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono,
1244 						     system_counterval.cycles);
1245 		nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw,
1246 						    system_counterval.cycles);
1247 	} while (read_seqcount_retry(&tk_core.seq, seq));
1248 
1249 	xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real);
1250 	xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw);
1251 
1252 	/*
1253 	 * Interpolate if necessary, adjusting back from the start of the
1254 	 * current interval
1255 	 */
1256 	if (do_interp) {
1257 		u64 partial_history_cycles, total_history_cycles;
1258 		bool discontinuity;
1259 
1260 		/*
1261 		 * Check that the counter value occurs after the provided
1262 		 * history reference and that the history doesn't cross a
1263 		 * clocksource change
1264 		 */
1265 		if (!history_begin ||
1266 		    !cycle_between(history_begin->cycles,
1267 				   system_counterval.cycles, cycles) ||
1268 		    history_begin->cs_was_changed_seq != cs_was_changed_seq)
1269 			return -EINVAL;
1270 		partial_history_cycles = cycles - system_counterval.cycles;
1271 		total_history_cycles = cycles - history_begin->cycles;
1272 		discontinuity =
1273 			history_begin->clock_was_set_seq != clock_was_set_seq;
1274 
1275 		ret = adjust_historical_crosststamp(history_begin,
1276 						    partial_history_cycles,
1277 						    total_history_cycles,
1278 						    discontinuity, xtstamp);
1279 		if (ret)
1280 			return ret;
1281 	}
1282 
1283 	return 0;
1284 }
1285 EXPORT_SYMBOL_GPL(get_device_system_crosststamp);
1286 
1287 /**
1288  * do_settimeofday64 - Sets the time of day.
1289  * @ts:     pointer to the timespec64 variable containing the new time
1290  *
1291  * Sets the time of day to the new time and update NTP and notify hrtimers
1292  */
1293 int do_settimeofday64(const struct timespec64 *ts)
1294 {
1295 	struct timekeeper *tk = &tk_core.timekeeper;
1296 	struct timespec64 ts_delta, xt;
1297 	unsigned long flags;
1298 	int ret = 0;
1299 
1300 	if (!timespec64_valid_settod(ts))
1301 		return -EINVAL;
1302 
1303 	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1304 	write_seqcount_begin(&tk_core.seq);
1305 
1306 	timekeeping_forward_now(tk);
1307 
1308 	xt = tk_xtime(tk);
1309 	ts_delta.tv_sec = ts->tv_sec - xt.tv_sec;
1310 	ts_delta.tv_nsec = ts->tv_nsec - xt.tv_nsec;
1311 
1312 	if (timespec64_compare(&tk->wall_to_monotonic, &ts_delta) > 0) {
1313 		ret = -EINVAL;
1314 		goto out;
1315 	}
1316 
1317 	tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts_delta));
1318 
1319 	tk_set_xtime(tk, ts);
1320 out:
1321 	timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1322 
1323 	write_seqcount_end(&tk_core.seq);
1324 	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1325 
1326 	/* signal hrtimers about time change */
1327 	clock_was_set();
1328 
1329 	if (!ret)
1330 		audit_tk_injoffset(ts_delta);
1331 
1332 	return ret;
1333 }
1334 EXPORT_SYMBOL(do_settimeofday64);
1335 
1336 /**
1337  * timekeeping_inject_offset - Adds or subtracts from the current time.
1338  * @ts:		Pointer to the timespec variable containing the offset
1339  *
1340  * Adds or subtracts an offset value from the current time.
1341  */
1342 static int timekeeping_inject_offset(const struct timespec64 *ts)
1343 {
1344 	struct timekeeper *tk = &tk_core.timekeeper;
1345 	unsigned long flags;
1346 	struct timespec64 tmp;
1347 	int ret = 0;
1348 
1349 	if (ts->tv_nsec < 0 || ts->tv_nsec >= NSEC_PER_SEC)
1350 		return -EINVAL;
1351 
1352 	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1353 	write_seqcount_begin(&tk_core.seq);
1354 
1355 	timekeeping_forward_now(tk);
1356 
1357 	/* Make sure the proposed value is valid */
1358 	tmp = timespec64_add(tk_xtime(tk), *ts);
1359 	if (timespec64_compare(&tk->wall_to_monotonic, ts) > 0 ||
1360 	    !timespec64_valid_settod(&tmp)) {
1361 		ret = -EINVAL;
1362 		goto error;
1363 	}
1364 
1365 	tk_xtime_add(tk, ts);
1366 	tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *ts));
1367 
1368 error: /* even if we error out, we forwarded the time, so call update */
1369 	timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1370 
1371 	write_seqcount_end(&tk_core.seq);
1372 	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1373 
1374 	/* signal hrtimers about time change */
1375 	clock_was_set();
1376 
1377 	return ret;
1378 }
1379 
1380 /*
1381  * Indicates if there is an offset between the system clock and the hardware
1382  * clock/persistent clock/rtc.
1383  */
1384 int persistent_clock_is_local;
1385 
1386 /*
1387  * Adjust the time obtained from the CMOS to be UTC time instead of
1388  * local time.
1389  *
1390  * This is ugly, but preferable to the alternatives.  Otherwise we
1391  * would either need to write a program to do it in /etc/rc (and risk
1392  * confusion if the program gets run more than once; it would also be
1393  * hard to make the program warp the clock precisely n hours)  or
1394  * compile in the timezone information into the kernel.  Bad, bad....
1395  *
1396  *						- TYT, 1992-01-01
1397  *
1398  * The best thing to do is to keep the CMOS clock in universal time (UTC)
1399  * as real UNIX machines always do it. This avoids all headaches about
1400  * daylight saving times and warping kernel clocks.
1401  */
1402 void timekeeping_warp_clock(void)
1403 {
1404 	if (sys_tz.tz_minuteswest != 0) {
1405 		struct timespec64 adjust;
1406 
1407 		persistent_clock_is_local = 1;
1408 		adjust.tv_sec = sys_tz.tz_minuteswest * 60;
1409 		adjust.tv_nsec = 0;
1410 		timekeeping_inject_offset(&adjust);
1411 	}
1412 }
1413 
1414 /*
1415  * __timekeeping_set_tai_offset - Sets the TAI offset from UTC and monotonic
1416  */
1417 static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
1418 {
1419 	tk->tai_offset = tai_offset;
1420 	tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0));
1421 }
1422 
1423 /*
1424  * change_clocksource - Swaps clocksources if a new one is available
1425  *
1426  * Accumulates current time interval and initializes new clocksource
1427  */
1428 static int change_clocksource(void *data)
1429 {
1430 	struct timekeeper *tk = &tk_core.timekeeper;
1431 	struct clocksource *new, *old;
1432 	unsigned long flags;
1433 
1434 	new = (struct clocksource *) data;
1435 
1436 	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1437 	write_seqcount_begin(&tk_core.seq);
1438 
1439 	timekeeping_forward_now(tk);
1440 	/*
1441 	 * If the cs is in module, get a module reference. Succeeds
1442 	 * for built-in code (owner == NULL) as well.
1443 	 */
1444 	if (try_module_get(new->owner)) {
1445 		if (!new->enable || new->enable(new) == 0) {
1446 			old = tk->tkr_mono.clock;
1447 			tk_setup_internals(tk, new);
1448 			if (old->disable)
1449 				old->disable(old);
1450 			module_put(old->owner);
1451 		} else {
1452 			module_put(new->owner);
1453 		}
1454 	}
1455 	timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1456 
1457 	write_seqcount_end(&tk_core.seq);
1458 	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1459 
1460 	return 0;
1461 }
1462 
1463 /**
1464  * timekeeping_notify - Install a new clock source
1465  * @clock:		pointer to the clock source
1466  *
1467  * This function is called from clocksource.c after a new, better clock
1468  * source has been registered. The caller holds the clocksource_mutex.
1469  */
1470 int timekeeping_notify(struct clocksource *clock)
1471 {
1472 	struct timekeeper *tk = &tk_core.timekeeper;
1473 
1474 	if (tk->tkr_mono.clock == clock)
1475 		return 0;
1476 	stop_machine(change_clocksource, clock, NULL);
1477 	tick_clock_notify();
1478 	return tk->tkr_mono.clock == clock ? 0 : -1;
1479 }
1480 
1481 /**
1482  * ktime_get_raw_ts64 - Returns the raw monotonic time in a timespec
1483  * @ts:		pointer to the timespec64 to be set
1484  *
1485  * Returns the raw monotonic time (completely un-modified by ntp)
1486  */
1487 void ktime_get_raw_ts64(struct timespec64 *ts)
1488 {
1489 	struct timekeeper *tk = &tk_core.timekeeper;
1490 	unsigned int seq;
1491 	u64 nsecs;
1492 
1493 	do {
1494 		seq = read_seqcount_begin(&tk_core.seq);
1495 		ts->tv_sec = tk->raw_sec;
1496 		nsecs = timekeeping_get_ns(&tk->tkr_raw);
1497 
1498 	} while (read_seqcount_retry(&tk_core.seq, seq));
1499 
1500 	ts->tv_nsec = 0;
1501 	timespec64_add_ns(ts, nsecs);
1502 }
1503 EXPORT_SYMBOL(ktime_get_raw_ts64);
1504 
1505 
1506 /**
1507  * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
1508  */
1509 int timekeeping_valid_for_hres(void)
1510 {
1511 	struct timekeeper *tk = &tk_core.timekeeper;
1512 	unsigned int seq;
1513 	int ret;
1514 
1515 	do {
1516 		seq = read_seqcount_begin(&tk_core.seq);
1517 
1518 		ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
1519 
1520 	} while (read_seqcount_retry(&tk_core.seq, seq));
1521 
1522 	return ret;
1523 }
1524 
1525 /**
1526  * timekeeping_max_deferment - Returns max time the clocksource can be deferred
1527  */
1528 u64 timekeeping_max_deferment(void)
1529 {
1530 	struct timekeeper *tk = &tk_core.timekeeper;
1531 	unsigned int seq;
1532 	u64 ret;
1533 
1534 	do {
1535 		seq = read_seqcount_begin(&tk_core.seq);
1536 
1537 		ret = tk->tkr_mono.clock->max_idle_ns;
1538 
1539 	} while (read_seqcount_retry(&tk_core.seq, seq));
1540 
1541 	return ret;
1542 }
1543 
1544 /**
1545  * read_persistent_clock64 -  Return time from the persistent clock.
1546  * @ts: Pointer to the storage for the readout value
1547  *
1548  * Weak dummy function for arches that do not yet support it.
1549  * Reads the time from the battery backed persistent clock.
1550  * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
1551  *
1552  *  XXX - Do be sure to remove it once all arches implement it.
1553  */
1554 void __weak read_persistent_clock64(struct timespec64 *ts)
1555 {
1556 	ts->tv_sec = 0;
1557 	ts->tv_nsec = 0;
1558 }
1559 
1560 /**
1561  * read_persistent_wall_and_boot_offset - Read persistent clock, and also offset
1562  *                                        from the boot.
1563  *
1564  * Weak dummy function for arches that do not yet support it.
1565  * @wall_time:	- current time as returned by persistent clock
1566  * @boot_offset: - offset that is defined as wall_time - boot_time
1567  *
1568  * The default function calculates offset based on the current value of
1569  * local_clock(). This way architectures that support sched_clock() but don't
1570  * support dedicated boot time clock will provide the best estimate of the
1571  * boot time.
1572  */
1573 void __weak __init
1574 read_persistent_wall_and_boot_offset(struct timespec64 *wall_time,
1575 				     struct timespec64 *boot_offset)
1576 {
1577 	read_persistent_clock64(wall_time);
1578 	*boot_offset = ns_to_timespec64(local_clock());
1579 }
1580 
1581 /*
1582  * Flag reflecting whether timekeeping_resume() has injected sleeptime.
1583  *
1584  * The flag starts of false and is only set when a suspend reaches
1585  * timekeeping_suspend(), timekeeping_resume() sets it to false when the
1586  * timekeeper clocksource is not stopping across suspend and has been
1587  * used to update sleep time. If the timekeeper clocksource has stopped
1588  * then the flag stays true and is used by the RTC resume code to decide
1589  * whether sleeptime must be injected and if so the flag gets false then.
1590  *
1591  * If a suspend fails before reaching timekeeping_resume() then the flag
1592  * stays false and prevents erroneous sleeptime injection.
1593  */
1594 static bool suspend_timing_needed;
1595 
1596 /* Flag for if there is a persistent clock on this platform */
1597 static bool persistent_clock_exists;
1598 
1599 /*
1600  * timekeeping_init - Initializes the clocksource and common timekeeping values
1601  */
1602 void __init timekeeping_init(void)
1603 {
1604 	struct timespec64 wall_time, boot_offset, wall_to_mono;
1605 	struct timekeeper *tk = &tk_core.timekeeper;
1606 	struct clocksource *clock;
1607 	unsigned long flags;
1608 
1609 	read_persistent_wall_and_boot_offset(&wall_time, &boot_offset);
1610 	if (timespec64_valid_settod(&wall_time) &&
1611 	    timespec64_to_ns(&wall_time) > 0) {
1612 		persistent_clock_exists = true;
1613 	} else if (timespec64_to_ns(&wall_time) != 0) {
1614 		pr_warn("Persistent clock returned invalid value");
1615 		wall_time = (struct timespec64){0};
1616 	}
1617 
1618 	if (timespec64_compare(&wall_time, &boot_offset) < 0)
1619 		boot_offset = (struct timespec64){0};
1620 
1621 	/*
1622 	 * We want set wall_to_mono, so the following is true:
1623 	 * wall time + wall_to_mono = boot time
1624 	 */
1625 	wall_to_mono = timespec64_sub(boot_offset, wall_time);
1626 
1627 	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1628 	write_seqcount_begin(&tk_core.seq);
1629 	ntp_init();
1630 
1631 	clock = clocksource_default_clock();
1632 	if (clock->enable)
1633 		clock->enable(clock);
1634 	tk_setup_internals(tk, clock);
1635 
1636 	tk_set_xtime(tk, &wall_time);
1637 	tk->raw_sec = 0;
1638 
1639 	tk_set_wall_to_mono(tk, wall_to_mono);
1640 
1641 	timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
1642 
1643 	write_seqcount_end(&tk_core.seq);
1644 	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1645 }
1646 
1647 /* time in seconds when suspend began for persistent clock */
1648 static struct timespec64 timekeeping_suspend_time;
1649 
1650 /**
1651  * __timekeeping_inject_sleeptime - Internal function to add sleep interval
1652  * @tk:		Pointer to the timekeeper to be updated
1653  * @delta:	Pointer to the delta value in timespec64 format
1654  *
1655  * Takes a timespec offset measuring a suspend interval and properly
1656  * adds the sleep offset to the timekeeping variables.
1657  */
1658 static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
1659 					   const struct timespec64 *delta)
1660 {
1661 	if (!timespec64_valid_strict(delta)) {
1662 		printk_deferred(KERN_WARNING
1663 				"__timekeeping_inject_sleeptime: Invalid "
1664 				"sleep delta value!\n");
1665 		return;
1666 	}
1667 	tk_xtime_add(tk, delta);
1668 	tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta));
1669 	tk_update_sleep_time(tk, timespec64_to_ktime(*delta));
1670 	tk_debug_account_sleep_time(delta);
1671 }
1672 
1673 #if defined(CONFIG_PM_SLEEP) && defined(CONFIG_RTC_HCTOSYS_DEVICE)
1674 /**
1675  * We have three kinds of time sources to use for sleep time
1676  * injection, the preference order is:
1677  * 1) non-stop clocksource
1678  * 2) persistent clock (ie: RTC accessible when irqs are off)
1679  * 3) RTC
1680  *
1681  * 1) and 2) are used by timekeeping, 3) by RTC subsystem.
1682  * If system has neither 1) nor 2), 3) will be used finally.
1683  *
1684  *
1685  * If timekeeping has injected sleeptime via either 1) or 2),
1686  * 3) becomes needless, so in this case we don't need to call
1687  * rtc_resume(), and this is what timekeeping_rtc_skipresume()
1688  * means.
1689  */
1690 bool timekeeping_rtc_skipresume(void)
1691 {
1692 	return !suspend_timing_needed;
1693 }
1694 
1695 /**
1696  * 1) can be determined whether to use or not only when doing
1697  * timekeeping_resume() which is invoked after rtc_suspend(),
1698  * so we can't skip rtc_suspend() surely if system has 1).
1699  *
1700  * But if system has 2), 2) will definitely be used, so in this
1701  * case we don't need to call rtc_suspend(), and this is what
1702  * timekeeping_rtc_skipsuspend() means.
1703  */
1704 bool timekeeping_rtc_skipsuspend(void)
1705 {
1706 	return persistent_clock_exists;
1707 }
1708 
1709 /**
1710  * timekeeping_inject_sleeptime64 - Adds suspend interval to timeekeeping values
1711  * @delta: pointer to a timespec64 delta value
1712  *
1713  * This hook is for architectures that cannot support read_persistent_clock64
1714  * because their RTC/persistent clock is only accessible when irqs are enabled.
1715  * and also don't have an effective nonstop clocksource.
1716  *
1717  * This function should only be called by rtc_resume(), and allows
1718  * a suspend offset to be injected into the timekeeping values.
1719  */
1720 void timekeeping_inject_sleeptime64(const struct timespec64 *delta)
1721 {
1722 	struct timekeeper *tk = &tk_core.timekeeper;
1723 	unsigned long flags;
1724 
1725 	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1726 	write_seqcount_begin(&tk_core.seq);
1727 
1728 	suspend_timing_needed = false;
1729 
1730 	timekeeping_forward_now(tk);
1731 
1732 	__timekeeping_inject_sleeptime(tk, delta);
1733 
1734 	timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1735 
1736 	write_seqcount_end(&tk_core.seq);
1737 	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1738 
1739 	/* signal hrtimers about time change */
1740 	clock_was_set();
1741 }
1742 #endif
1743 
1744 /**
1745  * timekeeping_resume - Resumes the generic timekeeping subsystem.
1746  */
1747 void timekeeping_resume(void)
1748 {
1749 	struct timekeeper *tk = &tk_core.timekeeper;
1750 	struct clocksource *clock = tk->tkr_mono.clock;
1751 	unsigned long flags;
1752 	struct timespec64 ts_new, ts_delta;
1753 	u64 cycle_now, nsec;
1754 	bool inject_sleeptime = false;
1755 
1756 	read_persistent_clock64(&ts_new);
1757 
1758 	clockevents_resume();
1759 	clocksource_resume();
1760 
1761 	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1762 	write_seqcount_begin(&tk_core.seq);
1763 
1764 	/*
1765 	 * After system resumes, we need to calculate the suspended time and
1766 	 * compensate it for the OS time. There are 3 sources that could be
1767 	 * used: Nonstop clocksource during suspend, persistent clock and rtc
1768 	 * device.
1769 	 *
1770 	 * One specific platform may have 1 or 2 or all of them, and the
1771 	 * preference will be:
1772 	 *	suspend-nonstop clocksource -> persistent clock -> rtc
1773 	 * The less preferred source will only be tried if there is no better
1774 	 * usable source. The rtc part is handled separately in rtc core code.
1775 	 */
1776 	cycle_now = tk_clock_read(&tk->tkr_mono);
1777 	nsec = clocksource_stop_suspend_timing(clock, cycle_now);
1778 	if (nsec > 0) {
1779 		ts_delta = ns_to_timespec64(nsec);
1780 		inject_sleeptime = true;
1781 	} else if (timespec64_compare(&ts_new, &timekeeping_suspend_time) > 0) {
1782 		ts_delta = timespec64_sub(ts_new, timekeeping_suspend_time);
1783 		inject_sleeptime = true;
1784 	}
1785 
1786 	if (inject_sleeptime) {
1787 		suspend_timing_needed = false;
1788 		__timekeeping_inject_sleeptime(tk, &ts_delta);
1789 	}
1790 
1791 	/* Re-base the last cycle value */
1792 	tk->tkr_mono.cycle_last = cycle_now;
1793 	tk->tkr_raw.cycle_last  = cycle_now;
1794 
1795 	tk->ntp_error = 0;
1796 	timekeeping_suspended = 0;
1797 	timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
1798 	write_seqcount_end(&tk_core.seq);
1799 	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1800 
1801 	touch_softlockup_watchdog();
1802 
1803 	tick_resume();
1804 	hrtimers_resume();
1805 }
1806 
1807 int timekeeping_suspend(void)
1808 {
1809 	struct timekeeper *tk = &tk_core.timekeeper;
1810 	unsigned long flags;
1811 	struct timespec64		delta, delta_delta;
1812 	static struct timespec64	old_delta;
1813 	struct clocksource *curr_clock;
1814 	u64 cycle_now;
1815 
1816 	read_persistent_clock64(&timekeeping_suspend_time);
1817 
1818 	/*
1819 	 * On some systems the persistent_clock can not be detected at
1820 	 * timekeeping_init by its return value, so if we see a valid
1821 	 * value returned, update the persistent_clock_exists flag.
1822 	 */
1823 	if (timekeeping_suspend_time.tv_sec || timekeeping_suspend_time.tv_nsec)
1824 		persistent_clock_exists = true;
1825 
1826 	suspend_timing_needed = true;
1827 
1828 	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1829 	write_seqcount_begin(&tk_core.seq);
1830 	timekeeping_forward_now(tk);
1831 	timekeeping_suspended = 1;
1832 
1833 	/*
1834 	 * Since we've called forward_now, cycle_last stores the value
1835 	 * just read from the current clocksource. Save this to potentially
1836 	 * use in suspend timing.
1837 	 */
1838 	curr_clock = tk->tkr_mono.clock;
1839 	cycle_now = tk->tkr_mono.cycle_last;
1840 	clocksource_start_suspend_timing(curr_clock, cycle_now);
1841 
1842 	if (persistent_clock_exists) {
1843 		/*
1844 		 * To avoid drift caused by repeated suspend/resumes,
1845 		 * which each can add ~1 second drift error,
1846 		 * try to compensate so the difference in system time
1847 		 * and persistent_clock time stays close to constant.
1848 		 */
1849 		delta = timespec64_sub(tk_xtime(tk), timekeeping_suspend_time);
1850 		delta_delta = timespec64_sub(delta, old_delta);
1851 		if (abs(delta_delta.tv_sec) >= 2) {
1852 			/*
1853 			 * if delta_delta is too large, assume time correction
1854 			 * has occurred and set old_delta to the current delta.
1855 			 */
1856 			old_delta = delta;
1857 		} else {
1858 			/* Otherwise try to adjust old_system to compensate */
1859 			timekeeping_suspend_time =
1860 				timespec64_add(timekeeping_suspend_time, delta_delta);
1861 		}
1862 	}
1863 
1864 	timekeeping_update(tk, TK_MIRROR);
1865 	halt_fast_timekeeper(tk);
1866 	write_seqcount_end(&tk_core.seq);
1867 	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1868 
1869 	tick_suspend();
1870 	clocksource_suspend();
1871 	clockevents_suspend();
1872 
1873 	return 0;
1874 }
1875 
1876 /* sysfs resume/suspend bits for timekeeping */
1877 static struct syscore_ops timekeeping_syscore_ops = {
1878 	.resume		= timekeeping_resume,
1879 	.suspend	= timekeeping_suspend,
1880 };
1881 
1882 static int __init timekeeping_init_ops(void)
1883 {
1884 	register_syscore_ops(&timekeeping_syscore_ops);
1885 	return 0;
1886 }
1887 device_initcall(timekeeping_init_ops);
1888 
1889 /*
1890  * Apply a multiplier adjustment to the timekeeper
1891  */
1892 static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
1893 							 s64 offset,
1894 							 s32 mult_adj)
1895 {
1896 	s64 interval = tk->cycle_interval;
1897 
1898 	if (mult_adj == 0) {
1899 		return;
1900 	} else if (mult_adj == -1) {
1901 		interval = -interval;
1902 		offset = -offset;
1903 	} else if (mult_adj != 1) {
1904 		interval *= mult_adj;
1905 		offset *= mult_adj;
1906 	}
1907 
1908 	/*
1909 	 * So the following can be confusing.
1910 	 *
1911 	 * To keep things simple, lets assume mult_adj == 1 for now.
1912 	 *
1913 	 * When mult_adj != 1, remember that the interval and offset values
1914 	 * have been appropriately scaled so the math is the same.
1915 	 *
1916 	 * The basic idea here is that we're increasing the multiplier
1917 	 * by one, this causes the xtime_interval to be incremented by
1918 	 * one cycle_interval. This is because:
1919 	 *	xtime_interval = cycle_interval * mult
1920 	 * So if mult is being incremented by one:
1921 	 *	xtime_interval = cycle_interval * (mult + 1)
1922 	 * Its the same as:
1923 	 *	xtime_interval = (cycle_interval * mult) + cycle_interval
1924 	 * Which can be shortened to:
1925 	 *	xtime_interval += cycle_interval
1926 	 *
1927 	 * So offset stores the non-accumulated cycles. Thus the current
1928 	 * time (in shifted nanoseconds) is:
1929 	 *	now = (offset * adj) + xtime_nsec
1930 	 * Now, even though we're adjusting the clock frequency, we have
1931 	 * to keep time consistent. In other words, we can't jump back
1932 	 * in time, and we also want to avoid jumping forward in time.
1933 	 *
1934 	 * So given the same offset value, we need the time to be the same
1935 	 * both before and after the freq adjustment.
1936 	 *	now = (offset * adj_1) + xtime_nsec_1
1937 	 *	now = (offset * adj_2) + xtime_nsec_2
1938 	 * So:
1939 	 *	(offset * adj_1) + xtime_nsec_1 =
1940 	 *		(offset * adj_2) + xtime_nsec_2
1941 	 * And we know:
1942 	 *	adj_2 = adj_1 + 1
1943 	 * So:
1944 	 *	(offset * adj_1) + xtime_nsec_1 =
1945 	 *		(offset * (adj_1+1)) + xtime_nsec_2
1946 	 *	(offset * adj_1) + xtime_nsec_1 =
1947 	 *		(offset * adj_1) + offset + xtime_nsec_2
1948 	 * Canceling the sides:
1949 	 *	xtime_nsec_1 = offset + xtime_nsec_2
1950 	 * Which gives us:
1951 	 *	xtime_nsec_2 = xtime_nsec_1 - offset
1952 	 * Which simplfies to:
1953 	 *	xtime_nsec -= offset
1954 	 */
1955 	if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) {
1956 		/* NTP adjustment caused clocksource mult overflow */
1957 		WARN_ON_ONCE(1);
1958 		return;
1959 	}
1960 
1961 	tk->tkr_mono.mult += mult_adj;
1962 	tk->xtime_interval += interval;
1963 	tk->tkr_mono.xtime_nsec -= offset;
1964 }
1965 
1966 /*
1967  * Adjust the timekeeper's multiplier to the correct frequency
1968  * and also to reduce the accumulated error value.
1969  */
1970 static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
1971 {
1972 	u32 mult;
1973 
1974 	/*
1975 	 * Determine the multiplier from the current NTP tick length.
1976 	 * Avoid expensive division when the tick length doesn't change.
1977 	 */
1978 	if (likely(tk->ntp_tick == ntp_tick_length())) {
1979 		mult = tk->tkr_mono.mult - tk->ntp_err_mult;
1980 	} else {
1981 		tk->ntp_tick = ntp_tick_length();
1982 		mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) -
1983 				 tk->xtime_remainder, tk->cycle_interval);
1984 	}
1985 
1986 	/*
1987 	 * If the clock is behind the NTP time, increase the multiplier by 1
1988 	 * to catch up with it. If it's ahead and there was a remainder in the
1989 	 * tick division, the clock will slow down. Otherwise it will stay
1990 	 * ahead until the tick length changes to a non-divisible value.
1991 	 */
1992 	tk->ntp_err_mult = tk->ntp_error > 0 ? 1 : 0;
1993 	mult += tk->ntp_err_mult;
1994 
1995 	timekeeping_apply_adjustment(tk, offset, mult - tk->tkr_mono.mult);
1996 
1997 	if (unlikely(tk->tkr_mono.clock->maxadj &&
1998 		(abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult)
1999 			> tk->tkr_mono.clock->maxadj))) {
2000 		printk_once(KERN_WARNING
2001 			"Adjusting %s more than 11%% (%ld vs %ld)\n",
2002 			tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult,
2003 			(long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj);
2004 	}
2005 
2006 	/*
2007 	 * It may be possible that when we entered this function, xtime_nsec
2008 	 * was very small.  Further, if we're slightly speeding the clocksource
2009 	 * in the code above, its possible the required corrective factor to
2010 	 * xtime_nsec could cause it to underflow.
2011 	 *
2012 	 * Now, since we have already accumulated the second and the NTP
2013 	 * subsystem has been notified via second_overflow(), we need to skip
2014 	 * the next update.
2015 	 */
2016 	if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) {
2017 		tk->tkr_mono.xtime_nsec += (u64)NSEC_PER_SEC <<
2018 							tk->tkr_mono.shift;
2019 		tk->xtime_sec--;
2020 		tk->skip_second_overflow = 1;
2021 	}
2022 }
2023 
2024 /*
2025  * accumulate_nsecs_to_secs - Accumulates nsecs into secs
2026  *
2027  * Helper function that accumulates the nsecs greater than a second
2028  * from the xtime_nsec field to the xtime_secs field.
2029  * It also calls into the NTP code to handle leapsecond processing.
2030  */
2031 static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
2032 {
2033 	u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
2034 	unsigned int clock_set = 0;
2035 
2036 	while (tk->tkr_mono.xtime_nsec >= nsecps) {
2037 		int leap;
2038 
2039 		tk->tkr_mono.xtime_nsec -= nsecps;
2040 		tk->xtime_sec++;
2041 
2042 		/*
2043 		 * Skip NTP update if this second was accumulated before,
2044 		 * i.e. xtime_nsec underflowed in timekeeping_adjust()
2045 		 */
2046 		if (unlikely(tk->skip_second_overflow)) {
2047 			tk->skip_second_overflow = 0;
2048 			continue;
2049 		}
2050 
2051 		/* Figure out if its a leap sec and apply if needed */
2052 		leap = second_overflow(tk->xtime_sec);
2053 		if (unlikely(leap)) {
2054 			struct timespec64 ts;
2055 
2056 			tk->xtime_sec += leap;
2057 
2058 			ts.tv_sec = leap;
2059 			ts.tv_nsec = 0;
2060 			tk_set_wall_to_mono(tk,
2061 				timespec64_sub(tk->wall_to_monotonic, ts));
2062 
2063 			__timekeeping_set_tai_offset(tk, tk->tai_offset - leap);
2064 
2065 			clock_set = TK_CLOCK_WAS_SET;
2066 		}
2067 	}
2068 	return clock_set;
2069 }
2070 
2071 /*
2072  * logarithmic_accumulation - shifted accumulation of cycles
2073  *
2074  * This functions accumulates a shifted interval of cycles into
2075  * a shifted interval nanoseconds. Allows for O(log) accumulation
2076  * loop.
2077  *
2078  * Returns the unconsumed cycles.
2079  */
2080 static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset,
2081 				    u32 shift, unsigned int *clock_set)
2082 {
2083 	u64 interval = tk->cycle_interval << shift;
2084 	u64 snsec_per_sec;
2085 
2086 	/* If the offset is smaller than a shifted interval, do nothing */
2087 	if (offset < interval)
2088 		return offset;
2089 
2090 	/* Accumulate one shifted interval */
2091 	offset -= interval;
2092 	tk->tkr_mono.cycle_last += interval;
2093 	tk->tkr_raw.cycle_last  += interval;
2094 
2095 	tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift;
2096 	*clock_set |= accumulate_nsecs_to_secs(tk);
2097 
2098 	/* Accumulate raw time */
2099 	tk->tkr_raw.xtime_nsec += tk->raw_interval << shift;
2100 	snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
2101 	while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) {
2102 		tk->tkr_raw.xtime_nsec -= snsec_per_sec;
2103 		tk->raw_sec++;
2104 	}
2105 
2106 	/* Accumulate error between NTP and clock interval */
2107 	tk->ntp_error += tk->ntp_tick << shift;
2108 	tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
2109 						(tk->ntp_error_shift + shift);
2110 
2111 	return offset;
2112 }
2113 
2114 /*
2115  * timekeeping_advance - Updates the timekeeper to the current time and
2116  * current NTP tick length
2117  */
2118 static void timekeeping_advance(enum timekeeping_adv_mode mode)
2119 {
2120 	struct timekeeper *real_tk = &tk_core.timekeeper;
2121 	struct timekeeper *tk = &shadow_timekeeper;
2122 	u64 offset;
2123 	int shift = 0, maxshift;
2124 	unsigned int clock_set = 0;
2125 	unsigned long flags;
2126 
2127 	raw_spin_lock_irqsave(&timekeeper_lock, flags);
2128 
2129 	/* Make sure we're fully resumed: */
2130 	if (unlikely(timekeeping_suspended))
2131 		goto out;
2132 
2133 	offset = clocksource_delta(tk_clock_read(&tk->tkr_mono),
2134 				   tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
2135 
2136 	/* Check if there's really nothing to do */
2137 	if (offset < real_tk->cycle_interval && mode == TK_ADV_TICK)
2138 		goto out;
2139 
2140 	/* Do some additional sanity checking */
2141 	timekeeping_check_update(tk, offset);
2142 
2143 	/*
2144 	 * With NO_HZ we may have to accumulate many cycle_intervals
2145 	 * (think "ticks") worth of time at once. To do this efficiently,
2146 	 * we calculate the largest doubling multiple of cycle_intervals
2147 	 * that is smaller than the offset.  We then accumulate that
2148 	 * chunk in one go, and then try to consume the next smaller
2149 	 * doubled multiple.
2150 	 */
2151 	shift = ilog2(offset) - ilog2(tk->cycle_interval);
2152 	shift = max(0, shift);
2153 	/* Bound shift to one less than what overflows tick_length */
2154 	maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
2155 	shift = min(shift, maxshift);
2156 	while (offset >= tk->cycle_interval) {
2157 		offset = logarithmic_accumulation(tk, offset, shift,
2158 							&clock_set);
2159 		if (offset < tk->cycle_interval<<shift)
2160 			shift--;
2161 	}
2162 
2163 	/* Adjust the multiplier to correct NTP error */
2164 	timekeeping_adjust(tk, offset);
2165 
2166 	/*
2167 	 * Finally, make sure that after the rounding
2168 	 * xtime_nsec isn't larger than NSEC_PER_SEC
2169 	 */
2170 	clock_set |= accumulate_nsecs_to_secs(tk);
2171 
2172 	write_seqcount_begin(&tk_core.seq);
2173 	/*
2174 	 * Update the real timekeeper.
2175 	 *
2176 	 * We could avoid this memcpy by switching pointers, but that
2177 	 * requires changes to all other timekeeper usage sites as
2178 	 * well, i.e. move the timekeeper pointer getter into the
2179 	 * spinlocked/seqcount protected sections. And we trade this
2180 	 * memcpy under the tk_core.seq against one before we start
2181 	 * updating.
2182 	 */
2183 	timekeeping_update(tk, clock_set);
2184 	memcpy(real_tk, tk, sizeof(*tk));
2185 	/* The memcpy must come last. Do not put anything here! */
2186 	write_seqcount_end(&tk_core.seq);
2187 out:
2188 	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2189 	if (clock_set)
2190 		/* Have to call _delayed version, since in irq context*/
2191 		clock_was_set_delayed();
2192 }
2193 
2194 /**
2195  * update_wall_time - Uses the current clocksource to increment the wall time
2196  *
2197  */
2198 void update_wall_time(void)
2199 {
2200 	timekeeping_advance(TK_ADV_TICK);
2201 }
2202 
2203 /**
2204  * getboottime64 - Return the real time of system boot.
2205  * @ts:		pointer to the timespec64 to be set
2206  *
2207  * Returns the wall-time of boot in a timespec64.
2208  *
2209  * This is based on the wall_to_monotonic offset and the total suspend
2210  * time. Calls to settimeofday will affect the value returned (which
2211  * basically means that however wrong your real time clock is at boot time,
2212  * you get the right time here).
2213  */
2214 void getboottime64(struct timespec64 *ts)
2215 {
2216 	struct timekeeper *tk = &tk_core.timekeeper;
2217 	ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot);
2218 
2219 	*ts = ktime_to_timespec64(t);
2220 }
2221 EXPORT_SYMBOL_GPL(getboottime64);
2222 
2223 void ktime_get_coarse_real_ts64(struct timespec64 *ts)
2224 {
2225 	struct timekeeper *tk = &tk_core.timekeeper;
2226 	unsigned int seq;
2227 
2228 	do {
2229 		seq = read_seqcount_begin(&tk_core.seq);
2230 
2231 		*ts = tk_xtime(tk);
2232 	} while (read_seqcount_retry(&tk_core.seq, seq));
2233 }
2234 EXPORT_SYMBOL(ktime_get_coarse_real_ts64);
2235 
2236 void ktime_get_coarse_ts64(struct timespec64 *ts)
2237 {
2238 	struct timekeeper *tk = &tk_core.timekeeper;
2239 	struct timespec64 now, mono;
2240 	unsigned int seq;
2241 
2242 	do {
2243 		seq = read_seqcount_begin(&tk_core.seq);
2244 
2245 		now = tk_xtime(tk);
2246 		mono = tk->wall_to_monotonic;
2247 	} while (read_seqcount_retry(&tk_core.seq, seq));
2248 
2249 	set_normalized_timespec64(ts, now.tv_sec + mono.tv_sec,
2250 				now.tv_nsec + mono.tv_nsec);
2251 }
2252 EXPORT_SYMBOL(ktime_get_coarse_ts64);
2253 
2254 /*
2255  * Must hold jiffies_lock
2256  */
2257 void do_timer(unsigned long ticks)
2258 {
2259 	jiffies_64 += ticks;
2260 	calc_global_load();
2261 }
2262 
2263 /**
2264  * ktime_get_update_offsets_now - hrtimer helper
2265  * @cwsseq:	pointer to check and store the clock was set sequence number
2266  * @offs_real:	pointer to storage for monotonic -> realtime offset
2267  * @offs_boot:	pointer to storage for monotonic -> boottime offset
2268  * @offs_tai:	pointer to storage for monotonic -> clock tai offset
2269  *
2270  * Returns current monotonic time and updates the offsets if the
2271  * sequence number in @cwsseq and timekeeper.clock_was_set_seq are
2272  * different.
2273  *
2274  * Called from hrtimer_interrupt() or retrigger_next_event()
2275  */
2276 ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real,
2277 				     ktime_t *offs_boot, ktime_t *offs_tai)
2278 {
2279 	struct timekeeper *tk = &tk_core.timekeeper;
2280 	unsigned int seq;
2281 	ktime_t base;
2282 	u64 nsecs;
2283 
2284 	do {
2285 		seq = read_seqcount_begin(&tk_core.seq);
2286 
2287 		base = tk->tkr_mono.base;
2288 		nsecs = timekeeping_get_ns(&tk->tkr_mono);
2289 		base = ktime_add_ns(base, nsecs);
2290 
2291 		if (*cwsseq != tk->clock_was_set_seq) {
2292 			*cwsseq = tk->clock_was_set_seq;
2293 			*offs_real = tk->offs_real;
2294 			*offs_boot = tk->offs_boot;
2295 			*offs_tai = tk->offs_tai;
2296 		}
2297 
2298 		/* Handle leapsecond insertion adjustments */
2299 		if (unlikely(base >= tk->next_leap_ktime))
2300 			*offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0));
2301 
2302 	} while (read_seqcount_retry(&tk_core.seq, seq));
2303 
2304 	return base;
2305 }
2306 
2307 /*
2308  * timekeeping_validate_timex - Ensures the timex is ok for use in do_adjtimex
2309  */
2310 static int timekeeping_validate_timex(const struct __kernel_timex *txc)
2311 {
2312 	if (txc->modes & ADJ_ADJTIME) {
2313 		/* singleshot must not be used with any other mode bits */
2314 		if (!(txc->modes & ADJ_OFFSET_SINGLESHOT))
2315 			return -EINVAL;
2316 		if (!(txc->modes & ADJ_OFFSET_READONLY) &&
2317 		    !capable(CAP_SYS_TIME))
2318 			return -EPERM;
2319 	} else {
2320 		/* In order to modify anything, you gotta be super-user! */
2321 		if (txc->modes && !capable(CAP_SYS_TIME))
2322 			return -EPERM;
2323 		/*
2324 		 * if the quartz is off by more than 10% then
2325 		 * something is VERY wrong!
2326 		 */
2327 		if (txc->modes & ADJ_TICK &&
2328 		    (txc->tick <  900000/USER_HZ ||
2329 		     txc->tick > 1100000/USER_HZ))
2330 			return -EINVAL;
2331 	}
2332 
2333 	if (txc->modes & ADJ_SETOFFSET) {
2334 		/* In order to inject time, you gotta be super-user! */
2335 		if (!capable(CAP_SYS_TIME))
2336 			return -EPERM;
2337 
2338 		/*
2339 		 * Validate if a timespec/timeval used to inject a time
2340 		 * offset is valid.  Offsets can be postive or negative, so
2341 		 * we don't check tv_sec. The value of the timeval/timespec
2342 		 * is the sum of its fields,but *NOTE*:
2343 		 * The field tv_usec/tv_nsec must always be non-negative and
2344 		 * we can't have more nanoseconds/microseconds than a second.
2345 		 */
2346 		if (txc->time.tv_usec < 0)
2347 			return -EINVAL;
2348 
2349 		if (txc->modes & ADJ_NANO) {
2350 			if (txc->time.tv_usec >= NSEC_PER_SEC)
2351 				return -EINVAL;
2352 		} else {
2353 			if (txc->time.tv_usec >= USEC_PER_SEC)
2354 				return -EINVAL;
2355 		}
2356 	}
2357 
2358 	/*
2359 	 * Check for potential multiplication overflows that can
2360 	 * only happen on 64-bit systems:
2361 	 */
2362 	if ((txc->modes & ADJ_FREQUENCY) && (BITS_PER_LONG == 64)) {
2363 		if (LLONG_MIN / PPM_SCALE > txc->freq)
2364 			return -EINVAL;
2365 		if (LLONG_MAX / PPM_SCALE < txc->freq)
2366 			return -EINVAL;
2367 	}
2368 
2369 	return 0;
2370 }
2371 
2372 
2373 /**
2374  * do_adjtimex() - Accessor function to NTP __do_adjtimex function
2375  */
2376 int do_adjtimex(struct __kernel_timex *txc)
2377 {
2378 	struct timekeeper *tk = &tk_core.timekeeper;
2379 	struct audit_ntp_data ad;
2380 	unsigned long flags;
2381 	struct timespec64 ts;
2382 	s32 orig_tai, tai;
2383 	int ret;
2384 
2385 	/* Validate the data before disabling interrupts */
2386 	ret = timekeeping_validate_timex(txc);
2387 	if (ret)
2388 		return ret;
2389 
2390 	if (txc->modes & ADJ_SETOFFSET) {
2391 		struct timespec64 delta;
2392 		delta.tv_sec  = txc->time.tv_sec;
2393 		delta.tv_nsec = txc->time.tv_usec;
2394 		if (!(txc->modes & ADJ_NANO))
2395 			delta.tv_nsec *= 1000;
2396 		ret = timekeeping_inject_offset(&delta);
2397 		if (ret)
2398 			return ret;
2399 
2400 		audit_tk_injoffset(delta);
2401 	}
2402 
2403 	audit_ntp_init(&ad);
2404 
2405 	ktime_get_real_ts64(&ts);
2406 
2407 	raw_spin_lock_irqsave(&timekeeper_lock, flags);
2408 	write_seqcount_begin(&tk_core.seq);
2409 
2410 	orig_tai = tai = tk->tai_offset;
2411 	ret = __do_adjtimex(txc, &ts, &tai, &ad);
2412 
2413 	if (tai != orig_tai) {
2414 		__timekeeping_set_tai_offset(tk, tai);
2415 		timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
2416 	}
2417 	tk_update_leap_state(tk);
2418 
2419 	write_seqcount_end(&tk_core.seq);
2420 	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2421 
2422 	audit_ntp_log(&ad);
2423 
2424 	/* Update the multiplier immediately if frequency was set directly */
2425 	if (txc->modes & (ADJ_FREQUENCY | ADJ_TICK))
2426 		timekeeping_advance(TK_ADV_FREQ);
2427 
2428 	if (tai != orig_tai)
2429 		clock_was_set();
2430 
2431 	ntp_notify_cmos_timer();
2432 
2433 	return ret;
2434 }
2435 
2436 #ifdef CONFIG_NTP_PPS
2437 /**
2438  * hardpps() - Accessor function to NTP __hardpps function
2439  */
2440 void hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts)
2441 {
2442 	unsigned long flags;
2443 
2444 	raw_spin_lock_irqsave(&timekeeper_lock, flags);
2445 	write_seqcount_begin(&tk_core.seq);
2446 
2447 	__hardpps(phase_ts, raw_ts);
2448 
2449 	write_seqcount_end(&tk_core.seq);
2450 	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2451 }
2452 EXPORT_SYMBOL(hardpps);
2453 #endif /* CONFIG_NTP_PPS */
2454