1 // SPDX-License-Identifier: GPL-2.0+
15 #include <linux/tick.h>
20 #include "tick-internal.h"
25 	u64 delta = clocksource_delta(end, start, cs->mask);  in cycles_to_nsec_safe()
27 	if (likely(delta < cs->max_cycles))  in cycles_to_nsec_safe()
28 		return clocksource_cyc2ns(delta, cs->mult, cs->shift);  in cycles_to_nsec_safe()
30 	return mul_u64_u32_shr(delta, cs->mult, cs->shift);  in cycles_to_nsec_safe()
34  * clocks_calc_mult_shift - calculate mult/shift factors for scaled math of clocks
39  * @maxsec:	guaranteed runtime conversion range in seconds
44  * @to and @from are frequency values in HZ. For clock sources @to is
48  * The @maxsec conversion range argument controls the time frame in
50  * calculated mult and shift factors. This guarantees that no 64bit
69 		sftacc--;  in clocks_calc_mult_shift()
76 	for (sft = 32; sft > 0; sft--) {  in clocks_calc_mult_shift()
88 /*[Clocksource internal variables]---------
92  *	used to calculate the suspend time.
98  *	Name of the user-specified clocksource.
116  * Also a default for cs->uncertainty_margin when registering clocks.
124  * a lower bound for cs->uncertainty_margin values when registering clocks.
172 	 * of the kernel, so there is no point in creating a more permanent  in clocksource_watchdog_work()
183 	cs->flags &= ~(CLOCK_SOURCE_VALID_FOR_HRES | CLOCK_SOURCE_WATCHDOG);  in __clocksource_unstable()
184 	cs->flags |= CLOCK_SOURCE_UNSTABLE;  in __clocksource_unstable()
188 	 * re-rate and re-select.  in __clocksource_unstable()
190 	if (list_empty(&cs->list)) {  in __clocksource_unstable()
191 		cs->rating = 0;  in __clocksource_unstable()
195 	if (cs->mark_unstable)  in __clocksource_unstable()
196 		cs->mark_unstable(cs);  in __clocksource_unstable()
204  * clocksource_mark_unstable - mark clocksource unstable via watchdog
208  * it defers demotion and re-selection to a kthread.
215 	if (!(cs->flags & CLOCK_SOURCE_UNSTABLE)) {  in clocksource_mark_unstable()
216 		if (!list_empty(&cs->list) && list_empty(&cs->wd_list))  in clocksource_mark_unstable()
217 			list_add(&cs->wd_list, &watchdog_list);  in clocksource_mark_unstable()
241 		*wdnow = watchdog->read(watchdog);  in cs_watchdog_read()
242 		*csnow = cs->read(cs);  in cs_watchdog_read()
243 		wd_end = watchdog->read(watchdog);  in cs_watchdog_read()
244 		wd_end2 = watchdog->read(watchdog);  in cs_watchdog_read()
251 					smp_processor_id(), watchdog->name, nretries);  in cs_watchdog_read()
257 		 * Now compute delay in consecutive watchdog read to see if  in cs_watchdog_read()
259 		 * significant delay in reading both clocksource and watchdog.  in cs_watchdog_read()
261 		 * If consecutive WD read-back delay > WATCHDOG_MAX_SKEW/2,  in cs_watchdog_read()
270 …warn("timekeeping watchdog on CPU%d: wd-%s-wd excessive read-back delay of %lldns vs. limit of %ld…  in cs_watchdog_read()
271 		smp_processor_id(), cs->name, wd_delay, WATCHDOG_MAX_SKEW, wd_seq_delay, nretries, cs->name);  in cs_watchdog_read()
275 	pr_info("timekeeping watchdog on CPU%d: %s wd-wd read-back delay of %lldns\n",  in cs_watchdog_read()
276 		smp_processor_id(), watchdog->name, wd_seq_delay);  in cs_watchdog_read()
277 	pr_info("wd-%s-wd read-back delay of %lldns, clock-skew test skipped!\n",  in cs_watchdog_read()
278 		cs->name, wd_delay);  in cs_watchdog_read()
298 	/* If no checking desired, or no other CPU to check, leave. */  in clocksource_verify_choose_cpus()
318 	 * and no replacement CPU is selected.  This gracefully handles  in clocksource_verify_choose_cpus()
324 		cpu = cpumask_next(cpu - 1, cpu_online_mask);  in clocksource_verify_choose_cpus()
339 	csnow_mid = cs->read(cs);  in clocksource_verify_one_cpu()
359 		pr_warn("Not enough CPUs to check clocksource '%s'.\n", cs->name);  in clocksource_verify_percpu()
364 		cs->name, testcpu, cpumask_pr_args(&cpus_chosen));  in clocksource_verify_percpu()
369 		csnow_begin = cs->read(cs);  in clocksource_verify_percpu()
371 		csnow_end = cs->read(cs);  in clocksource_verify_percpu()
372 		delta = (s64)((csnow_mid - csnow_begin) & cs->mask);  in clocksource_verify_percpu()
375 		delta = (csnow_end - csnow_mid) & cs->mask;  in clocksource_verify_percpu()
389 			cpumask_pr_args(&cpus_ahead), testcpu, cs->name);  in clocksource_verify_percpu()
392 			cpumask_pr_args(&cpus_behind), testcpu, cs->name);  in clocksource_verify_percpu()
394 		pr_warn("        CPU %d check durations %lldns - %lldns for clocksource %s.\n",  in clocksource_verify_percpu()
395 			testcpu, cs_nsec_min, cs_nsec_max, cs->name);  in clocksource_verify_percpu()
404 		cs->flags &= ~CLOCK_SOURCE_WATCHDOG;  in clocksource_reset_watchdog()
427 		if (cs->flags & CLOCK_SOURCE_UNSTABLE) {  in clocksource_watchdog()
445 		 * watchdog check. So give system some space and suspend the  in clocksource_watchdog()
451 			 * cs->last could keep unchanged for 5 minutes, reset  in clocksource_watchdog()
460 		if (!(cs->flags & CLOCK_SOURCE_WATCHDOG) ||  in clocksource_watchdog()
462 			cs->flags |= CLOCK_SOURCE_WATCHDOG;  in clocksource_watchdog()
463 			cs->wd_last = wdnow;  in clocksource_watchdog()
464 			cs->cs_last = csnow;  in clocksource_watchdog()
468 		wd_nsec = cycles_to_nsec_safe(watchdog, cs->wd_last, wdnow);  in clocksource_watchdog()
469 		cs_nsec = cycles_to_nsec_safe(cs, cs->cs_last, csnow);  in clocksource_watchdog()
470 		wdlast = cs->wd_last; /* save these in case we print them */  in clocksource_watchdog()
471 		cslast = cs->cs_last;  in clocksource_watchdog()
472 		cs->cs_last = csnow;  in clocksource_watchdog()
473 		cs->wd_last = wdnow;  in clocksource_watchdog()
480 		 * on account of ksoftirqd not getting to run in a timely  in clocksource_watchdog()
500 		md = cs->uncertainty_margin + watchdog->uncertainty_margin;  in clocksource_watchdog()
501 		if (abs(cs_nsec - wd_nsec) > md) {  in clocksource_watchdog()
507 				smp_processor_id(), cs->name);  in clocksource_watchdog()
509 				watchdog->name, wd_nsec, wdnow, wdlast, watchdog->mask);  in clocksource_watchdog()
511 				cs->name, cs_nsec, csnow, cslast, cs->mask);  in clocksource_watchdog()
512 			cs_wd_msec = div_s64_rem(cs_nsec - wd_nsec, 1000 * 1000, &wd_rem);  in clocksource_watchdog()
515 				cs->name, cs_nsec - wd_nsec, cs_wd_msec, watchdog->name, wd_nsec, wd_msec);  in clocksource_watchdog()
517 				pr_warn("                      '%s' is current clocksource.\n", cs->name);  in clocksource_watchdog()
519 …                     '%s' (not '%s') is current clocksource.\n", curr_clocksource->name, cs->name);  in clocksource_watchdog()
521 				pr_warn("                      No current clocksource.\n");  in clocksource_watchdog()
526 		if (cs == curr_clocksource && cs->tick_stable)  in clocksource_watchdog()
527 			cs->tick_stable(cs);  in clocksource_watchdog()
529 		if (!(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) &&  in clocksource_watchdog()
530 		    (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) &&  in clocksource_watchdog()
531 		    (watchdog->flags & CLOCK_SOURCE_IS_CONTINUOUS)) {  in clocksource_watchdog()
532 			/* Mark it valid for high-res. */  in clocksource_watchdog()
533 			cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;  in clocksource_watchdog()
547 			 * clocksource let the tick code know about  in clocksource_watchdog()
551 				cs->flags |= CLOCK_SOURCE_RESELECT;  in clocksource_watchdog()
611 	INIT_LIST_HEAD(&cs->wd_list);  in clocksource_enqueue_watchdog()
613 	if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) {  in clocksource_enqueue_watchdog()
615 		list_add(&cs->wd_list, &watchdog_list);  in clocksource_enqueue_watchdog()
616 		cs->flags &= ~CLOCK_SOURCE_WATCHDOG;  in clocksource_enqueue_watchdog()
619 		if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS)  in clocksource_enqueue_watchdog()
620 			cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;  in clocksource_enqueue_watchdog()
637 		if (cs->flags & CLOCK_SOURCE_MUST_VERIFY)  in clocksource_select_watchdog()
645 		if (!watchdog || cs->rating > watchdog->rating)  in clocksource_select_watchdog()
664 		if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) {  in clocksource_dequeue_watchdog()
666 			list_del_init(&cs->wd_list);  in clocksource_dequeue_watchdog()
679 	/* Do any required per-CPU skew verification. */  in __clocksource_watchdog_kthread()
681 	    curr_clocksource->flags & CLOCK_SOURCE_UNSTABLE &&  in __clocksource_watchdog_kthread()
682 	    curr_clocksource->flags & CLOCK_SOURCE_VERIFY_PERCPU)  in __clocksource_watchdog_kthread()
687 		if (cs->flags & CLOCK_SOURCE_UNSTABLE) {  in __clocksource_watchdog_kthread()
688 			list_del_init(&cs->wd_list);  in __clocksource_watchdog_kthread()
692 		if (cs->flags & CLOCK_SOURCE_RESELECT) {  in __clocksource_watchdog_kthread()
693 			cs->flags &= ~CLOCK_SOURCE_RESELECT;  in __clocksource_watchdog_kthread()
722 	if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS)  in clocksource_enqueue_watchdog()
723 		cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;  in clocksource_enqueue_watchdog()
746 	 * Skip the clocksource which will be stopped in suspend state.  in __clocksource_suspend_select()
748 	if (!(cs->flags & CLOCK_SOURCE_SUSPEND_NONSTOP))  in __clocksource_suspend_select()
752 	 * The nonstop clocksource can be selected as the suspend clocksource to  in __clocksource_suspend_select()
753 	 * calculate the suspend time, so it should not supply suspend/resume  in __clocksource_suspend_select()
754 	 * interfaces to suspend the nonstop clocksource when system suspends.  in __clocksource_suspend_select()
756 	if (cs->suspend || cs->resume) {  in __clocksource_suspend_select()
757 		pr_warn("Nonstop clocksource %s should not supply suspend/resume interfaces\n",  in __clocksource_suspend_select()
758 			cs->name);  in __clocksource_suspend_select()
762 	if (!suspend_clocksource || cs->rating > suspend_clocksource->rating)  in __clocksource_suspend_select()
767  * clocksource_suspend_select - Select the best clocksource for suspend timing
788  * clocksource_start_suspend_timing - Start measuring the suspend timing
792  * This function will save the start cycle values of suspend timer to calculate
793  * the suspend time when resuming system.
795  * This function is called late in the suspend process from timekeeping_suspend(),
796  * that means processes are frozen, non-boot cpus and interrupts are disabled
797  * now. It is therefore possible to start the suspend timer without taking the
806 	 * If current clocksource is the suspend timer, we should use the  in clocksource_start_suspend_timing()
808 	 * from suspend timer.  in clocksource_start_suspend_timing()
815 	if (suspend_clocksource->enable &&  in clocksource_start_suspend_timing()
816 	    suspend_clocksource->enable(suspend_clocksource)) {  in clocksource_start_suspend_timing()
817 		pr_warn_once("Failed to enable the non-suspend-able clocksource.\n");  in clocksource_start_suspend_timing()
821 	suspend_start = suspend_clocksource->read(suspend_clocksource);  in clocksource_start_suspend_timing()
825  * clocksource_stop_suspend_timing - Stop measuring the suspend timing
829  * This function will calculate the suspend time from suspend timer.
831  * Returns nanoseconds since suspend started, 0 if no usable suspend clocksource.
833  * This function is called early in the resume process from timekeeping_resume(),
834  * that means there is only one cpu, no processes are running and the interrupts
835  * are disabled. It is therefore possible to stop the suspend timer without
846 	 * If current clocksource is the suspend timer, we should use the  in clocksource_stop_suspend_timing()
848 	 * avoid same reading from suspend timer.  in clocksource_stop_suspend_timing()
853 		now = suspend_clocksource->read(suspend_clocksource);  in clocksource_stop_suspend_timing()
859 	 * Disable the suspend timer to save power if current clocksource is  in clocksource_stop_suspend_timing()
860 	 * not the suspend timer.  in clocksource_stop_suspend_timing()
862 	if (!clocksource_is_suspend(cs) && suspend_clocksource->disable)  in clocksource_stop_suspend_timing()
863 		suspend_clocksource->disable(suspend_clocksource);  in clocksource_stop_suspend_timing()
869  * clocksource_suspend - suspend the clocksource(s)
876 		if (cs->suspend)  in clocksource_suspend()
877 			cs->suspend(cs);  in clocksource_suspend()
881  * clocksource_resume - resume the clocksource(s)
888 		if (cs->resume)  in clocksource_resume()
889 			cs->resume(cs);  in clocksource_resume()
895  * clocksource_touch_watchdog - Update watchdog
899  * was stopped in code which holds watchdog_lock.
907  * clocksource_max_adjustment- Returns max adjustment amount
917 	ret = (u64)cs->mult * 11;  in clocksource_max_adjustment()
923  * clocks_calc_max_nsecs - Returns maximum nanoseconds that can be converted
931  * NOTE: This function includes a safety margin of 50%, in other words, we
934  * delayed timers or bad hardware, which might result in time intervals that
943 	 * cyc2ns() function without overflowing a 64-bit result.  in clocks_calc_max_nsecs()
955 	max_nsecs = clocksource_cyc2ns(max_cycles, mult - maxadj, shift);  in clocks_calc_max_nsecs()
968  * clocksource_update_max_deferment - Updates the clocksource max_idle_ns & max_cycles
974 	cs->max_idle_ns = clocks_calc_max_nsecs(cs->mult, cs->shift,  in clocksource_update_max_deferment()
975 						cs->maxadj, cs->mask,  in clocksource_update_max_deferment()
976 						&cs->max_cycles);  in clocksource_update_max_deferment()
994 		if (oneshot && !(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES))  in clocksource_find_best()
1018 		if (strcmp(cs->name, override_name) != 0)  in __clocksource_select()
1021 		 * Check to make sure we don't switch to a non-highres  in __clocksource_select()
1022 		 * capable clocksource if the tick code is in oneshot  in __clocksource_select()
1025 		if (!(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) && oneshot) {  in __clocksource_select()
1027 			if (cs->flags & CLOCK_SOURCE_UNSTABLE) {  in __clocksource_select()
1028 …pr_warn("Override clocksource %s is unstable and not HRT compatible - cannot switch while in HRT/N…  in __clocksource_select()
1029 					cs->name);  in __clocksource_select()
1036 				pr_info("Override clocksource %s is not currently HRT compatible - deferring\n",  in __clocksource_select()
1037 					cs->name);  in __clocksource_select()
1047 		pr_info("Switched to clocksource %s\n", best->name);  in __clocksource_select()
1053  * clocksource_select - Select the best clocksource available
1071  * clocksource_done_booting - Called near the end of core bootup
1102 		if (tmp->rating < cs->rating)  in clocksource_enqueue()
1104 		entry = &tmp->list;  in clocksource_enqueue()
1106 	list_add(&cs->list, entry);  in clocksource_enqueue()
1110  * __clocksource_update_freq_scale - Used update clocksource with new freq
1115  * This should only be called from the clocksource->enable() method.
1126 	 * Default clocksources are *special* and self-define their mult/shift.  in __clocksource_update_freq_scale()
1132 		 * wrapping around. For clocksources which have a mask > 32-bit  in __clocksource_update_freq_scale()
1135 		 * amount. That results in a shift value of 24 for a  in __clocksource_update_freq_scale()
1136 		 * clocksource with mask >= 40-bit and f >= 4GHz. That maps to  in __clocksource_update_freq_scale()
1139 		sec = cs->mask;  in __clocksource_update_freq_scale()
1144 		else if (sec > 600 && cs->mask > UINT_MAX)  in __clocksource_update_freq_scale()
1147 		clocks_calc_mult_shift(&cs->mult, &cs->shift, freq,  in __clocksource_update_freq_scale()
1153 	 * If both scale and freq are non-zero, calculate the clock  in __clocksource_update_freq_scale()
1156 	 * take the tens-of-milliseconds WATCHDOG_THRESHOLD value for the  in __clocksource_update_freq_scale()
1161 	if (scale && freq && !cs->uncertainty_margin) {  in __clocksource_update_freq_scale()
1162 		cs->uncertainty_margin = NSEC_PER_SEC / (scale * freq);  in __clocksource_update_freq_scale()
1163 		if (cs->uncertainty_margin < 2 * WATCHDOG_MAX_SKEW)  in __clocksource_update_freq_scale()
1164 			cs->uncertainty_margin = 2 * WATCHDOG_MAX_SKEW;  in __clocksource_update_freq_scale()
1165 	} else if (!cs->uncertainty_margin) {  in __clocksource_update_freq_scale()
1166 		cs->uncertainty_margin = WATCHDOG_THRESHOLD;  in __clocksource_update_freq_scale()
1168 	WARN_ON_ONCE(cs->uncertainty_margin < 2 * WATCHDOG_MAX_SKEW);  in __clocksource_update_freq_scale()
1174 	cs->maxadj = clocksource_max_adjustment(cs);  in __clocksource_update_freq_scale()
1175 	while (freq && ((cs->mult + cs->maxadj < cs->mult)  in __clocksource_update_freq_scale()
1176 		|| (cs->mult - cs->maxadj > cs->mult))) {  in __clocksource_update_freq_scale()
1177 		cs->mult >>= 1;  in __clocksource_update_freq_scale()
1178 		cs->shift--;  in __clocksource_update_freq_scale()
1179 		cs->maxadj = clocksource_max_adjustment(cs);  in __clocksource_update_freq_scale()
1183 	 * Only warn for *special* clocksources that self-define  in __clocksource_update_freq_scale()
1186 	WARN_ONCE(cs->mult + cs->maxadj < cs->mult,  in __clocksource_update_freq_scale()
1188 		cs->name);  in __clocksource_update_freq_scale()
1193 		cs->name, cs->mask, cs->max_cycles, cs->max_idle_ns);  in __clocksource_update_freq_scale()
1198  * __clocksource_register_scale - Used to install new clocksources
1203  * Returns -EBUSY if registration fails, zero otherwise.
1214 	if (WARN_ON_ONCE((unsigned int)cs->id >= CSID_MAX))  in __clocksource_register_scale()
1215 		cs->id = CSID_GENERIC;  in __clocksource_register_scale()
1216 	if (cs->vdso_clock_mode < 0 ||  in __clocksource_register_scale()
1217 	    cs->vdso_clock_mode >= VDSO_CLOCKMODE_MAX) {  in __clocksource_register_scale()
1219 			cs->name, cs->vdso_clock_mode);  in __clocksource_register_scale()
1220 		cs->vdso_clock_mode = VDSO_CLOCKMODE_NONE;  in __clocksource_register_scale()
1244 	list_del(&cs->list);  in __clocksource_change_rating()
1245 	cs->rating = rating;  in __clocksource_change_rating()
1250  * clocksource_change_rating - Change the rating of a registered clocksource
1281 			return -EBUSY;  in clocksource_unbind()
1288 			return -EBUSY;  in clocksource_unbind()
1293 		 * Select and try to install a replacement suspend clocksource.  in clocksource_unbind()
1294 		 * If no replacement suspend clocksource, we will just let the  in clocksource_unbind()
1295 		 * clocksource go and have no suspend clocksource.  in clocksource_unbind()
1302 	list_del_init(&cs->list);  in clocksource_unbind()
1309  * clocksource_unregister - remove a registered clocksource
1317 	if (!list_empty(&cs->list))  in clocksource_unregister()
1326  * current_clocksource_show - sysfs interface for current clocksource
1340 	count = snprintf(buf, PAGE_SIZE, "%s\n", curr_clocksource->name);  in current_clocksource_show()
1352 		return -EINVAL;  in sysfs_get_uname()
1355 	if (buf[cnt-1] == '\n')  in sysfs_get_uname()
1356 		cnt--;  in sysfs_get_uname()
1364  * current_clocksource_store - interface for manually overriding clocksource
1392  * unbind_clocksource_store - interface for manually unbinding clocksource
1412 	ret = -ENODEV;  in unbind_clocksource_store()
1415 		if (strcmp(cs->name, name))  in unbind_clocksource_store()
1427  * available_clocksource_show - sysfs interface for listing clocksource
1444 		 * Don't show non-HRES clocksource if the tick code is  in available_clocksource_show()
1445 		 * in one shot mode (highres=on or nohz=on)  in available_clocksource_show()
1448 		    (src->flags & CLOCK_SOURCE_VALID_FOR_HRES))  in available_clocksource_show()
1450 				  max((ssize_t)PAGE_SIZE - count, (ssize_t)0),  in available_clocksource_show()
1451 				  "%s ", src->name);  in available_clocksource_show()
1456 			  max((ssize_t)PAGE_SIZE - count, (ssize_t)0), "\n");  in available_clocksource_show()
1495  * boot_override_clocksource - boot clock override
1513  * boot_override_clock - Compatibility layer for deprecated boot option
1522 		pr_warn("clock=pmtmr is deprecated - use clocksource=acpi_pm\n");  in boot_override_clock()
1525 	pr_warn("clock= boot option is deprecated - use clocksource=xyz\n");  in boot_override_clock()