1250c2277SThomas Gleixner /* 2835c34a1SDave Jones * check TSC synchronization. 3250c2277SThomas Gleixner * 4250c2277SThomas Gleixner * Copyright (C) 2006, Red Hat, Inc., Ingo Molnar 5250c2277SThomas Gleixner * 6250c2277SThomas Gleixner * We check whether all boot CPUs have their TSC's synchronized, 7250c2277SThomas Gleixner * print a warning if not and turn off the TSC clock-source. 8250c2277SThomas Gleixner * 9250c2277SThomas Gleixner * The warp-check is point-to-point between two CPUs, the CPU 10250c2277SThomas Gleixner * initiating the bootup is the 'source CPU', the freshly booting 11250c2277SThomas Gleixner * CPU is the 'target CPU'. 12250c2277SThomas Gleixner * 13250c2277SThomas Gleixner * Only two CPUs may participate - they can enter in any order. 14250c2277SThomas Gleixner * ( The serial nature of the boot logic and the CPU hotplug lock 15250c2277SThomas Gleixner * protects against more than 2 CPUs entering this code. ) 16250c2277SThomas Gleixner */ 178b223bc7SThomas Gleixner #include <linux/topology.h> 18250c2277SThomas Gleixner #include <linux/spinlock.h> 19250c2277SThomas Gleixner #include <linux/kernel.h> 20250c2277SThomas Gleixner #include <linux/smp.h> 21250c2277SThomas Gleixner #include <linux/nmi.h> 22250c2277SThomas Gleixner #include <asm/tsc.h> 23250c2277SThomas Gleixner 248b223bc7SThomas Gleixner struct tsc_adjust { 258b223bc7SThomas Gleixner s64 bootval; 268b223bc7SThomas Gleixner s64 adjusted; 271d0095feSThomas Gleixner unsigned long nextcheck; 281d0095feSThomas Gleixner bool warned; 298b223bc7SThomas Gleixner }; 308b223bc7SThomas Gleixner 318b223bc7SThomas Gleixner static DEFINE_PER_CPU(struct tsc_adjust, tsc_adjust); 328b223bc7SThomas Gleixner 336a369583SThomas Gleixner void tsc_verify_tsc_adjust(bool resume) 341d0095feSThomas Gleixner { 351d0095feSThomas Gleixner struct tsc_adjust *adj = this_cpu_ptr(&tsc_adjust); 361d0095feSThomas Gleixner s64 curval; 371d0095feSThomas Gleixner 381d0095feSThomas Gleixner if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) 391d0095feSThomas Gleixner return; 401d0095feSThomas Gleixner 411d0095feSThomas Gleixner /* Rate limit the MSR check */ 426a369583SThomas Gleixner if (!resume && time_before(jiffies, adj->nextcheck)) 431d0095feSThomas Gleixner return; 441d0095feSThomas Gleixner 451d0095feSThomas Gleixner adj->nextcheck = jiffies + HZ; 461d0095feSThomas Gleixner 471d0095feSThomas Gleixner rdmsrl(MSR_IA32_TSC_ADJUST, curval); 481d0095feSThomas Gleixner if (adj->adjusted == curval) 491d0095feSThomas Gleixner return; 501d0095feSThomas Gleixner 511d0095feSThomas Gleixner /* Restore the original value */ 521d0095feSThomas Gleixner wrmsrl(MSR_IA32_TSC_ADJUST, adj->adjusted); 531d0095feSThomas Gleixner 546a369583SThomas Gleixner if (!adj->warned || resume) { 551d0095feSThomas Gleixner pr_warn(FW_BUG "TSC ADJUST differs: CPU%u %lld --> %lld. Restoring\n", 561d0095feSThomas Gleixner smp_processor_id(), adj->adjusted, curval); 571d0095feSThomas Gleixner adj->warned = true; 581d0095feSThomas Gleixner } 591d0095feSThomas Gleixner } 601d0095feSThomas Gleixner 615bae1562SThomas Gleixner static void tsc_sanitize_first_cpu(struct tsc_adjust *cur, s64 bootval, 625bae1562SThomas Gleixner unsigned int cpu, bool bootcpu) 635bae1562SThomas Gleixner { 645bae1562SThomas Gleixner /* 655bae1562SThomas Gleixner * First online CPU in a package stores the boot value in the 665bae1562SThomas Gleixner * adjustment value. This value might change later via the sync 675bae1562SThomas Gleixner * mechanism. If that fails we still can yell about boot values not 685bae1562SThomas Gleixner * being consistent. 695bae1562SThomas Gleixner * 705bae1562SThomas Gleixner * On the boot cpu we just force set the ADJUST value to 0 if it's 715bae1562SThomas Gleixner * non zero. We don't do that on non boot cpus because physical 725bae1562SThomas Gleixner * hotplug should have set the ADJUST register to a value > 0 so 735bae1562SThomas Gleixner * the TSC is in sync with the already running cpus. 745bae1562SThomas Gleixner * 755bae1562SThomas Gleixner * But we always force positive ADJUST values. Otherwise the TSC 768c9b9d87SThomas Gleixner * deadline timer creates an interrupt storm. We also have to 778c9b9d87SThomas Gleixner * prevent values > 0x7FFFFFFF as those wreckage the timer as well. 785bae1562SThomas Gleixner */ 798c9b9d87SThomas Gleixner if ((bootcpu && bootval != 0) || (!bootcpu && bootval < 0) || 808c9b9d87SThomas Gleixner (bootval > 0x7FFFFFFF)) { 8116588f65SThomas Gleixner pr_warn(FW_BUG "TSC ADJUST: CPU%u: %lld force to 0\n", cpu, 8216588f65SThomas Gleixner bootval); 835bae1562SThomas Gleixner wrmsrl(MSR_IA32_TSC_ADJUST, 0); 845bae1562SThomas Gleixner bootval = 0; 855bae1562SThomas Gleixner } 865bae1562SThomas Gleixner cur->adjusted = bootval; 875bae1562SThomas Gleixner } 885bae1562SThomas Gleixner 898b223bc7SThomas Gleixner #ifndef CONFIG_SMP 905bae1562SThomas Gleixner bool __init tsc_store_and_check_tsc_adjust(bool bootcpu) 918b223bc7SThomas Gleixner { 92b8365543SThomas Gleixner struct tsc_adjust *cur = this_cpu_ptr(&tsc_adjust); 938b223bc7SThomas Gleixner s64 bootval; 948b223bc7SThomas Gleixner 958b223bc7SThomas Gleixner if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) 96a36f5136SThomas Gleixner return false; 978b223bc7SThomas Gleixner 988b223bc7SThomas Gleixner rdmsrl(MSR_IA32_TSC_ADJUST, bootval); 998b223bc7SThomas Gleixner cur->bootval = bootval; 1001d0095feSThomas Gleixner cur->nextcheck = jiffies + HZ; 1015bae1562SThomas Gleixner tsc_sanitize_first_cpu(cur, bootval, smp_processor_id(), bootcpu); 102a36f5136SThomas Gleixner return false; 1038b223bc7SThomas Gleixner } 1048b223bc7SThomas Gleixner 1058b223bc7SThomas Gleixner #else /* !CONFIG_SMP */ 1068b223bc7SThomas Gleixner 1078b223bc7SThomas Gleixner /* 1088b223bc7SThomas Gleixner * Store and check the TSC ADJUST MSR if available 1098b223bc7SThomas Gleixner */ 1105bae1562SThomas Gleixner bool tsc_store_and_check_tsc_adjust(bool bootcpu) 1118b223bc7SThomas Gleixner { 1128b223bc7SThomas Gleixner struct tsc_adjust *ref, *cur = this_cpu_ptr(&tsc_adjust); 1138b223bc7SThomas Gleixner unsigned int refcpu, cpu = smp_processor_id(); 11431f8a651SThomas Gleixner struct cpumask *mask; 1158b223bc7SThomas Gleixner s64 bootval; 1168b223bc7SThomas Gleixner 1178b223bc7SThomas Gleixner if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) 118a36f5136SThomas Gleixner return false; 1198b223bc7SThomas Gleixner 1208b223bc7SThomas Gleixner rdmsrl(MSR_IA32_TSC_ADJUST, bootval); 1218b223bc7SThomas Gleixner cur->bootval = bootval; 1221d0095feSThomas Gleixner cur->nextcheck = jiffies + HZ; 1231d0095feSThomas Gleixner cur->warned = false; 1248b223bc7SThomas Gleixner 1258b223bc7SThomas Gleixner /* 1268b223bc7SThomas Gleixner * Check whether this CPU is the first in a package to come up. In 1278b223bc7SThomas Gleixner * this case do not check the boot value against another package 1285bae1562SThomas Gleixner * because the new package might have been physically hotplugged, 1295bae1562SThomas Gleixner * where TSC_ADJUST is expected to be different. When called on the 1305bae1562SThomas Gleixner * boot CPU topology_core_cpumask() might not be available yet. 1318b223bc7SThomas Gleixner */ 13231f8a651SThomas Gleixner mask = topology_core_cpumask(cpu); 13331f8a651SThomas Gleixner refcpu = mask ? cpumask_any_but(mask, cpu) : nr_cpu_ids; 1348b223bc7SThomas Gleixner 1358b223bc7SThomas Gleixner if (refcpu >= nr_cpu_ids) { 1365bae1562SThomas Gleixner tsc_sanitize_first_cpu(cur, bootval, smp_processor_id(), 1375bae1562SThomas Gleixner bootcpu); 138a36f5136SThomas Gleixner return false; 1398b223bc7SThomas Gleixner } 1408b223bc7SThomas Gleixner 1418b223bc7SThomas Gleixner ref = per_cpu_ptr(&tsc_adjust, refcpu); 1428b223bc7SThomas Gleixner /* 1438b223bc7SThomas Gleixner * Compare the boot value and complain if it differs in the 1448b223bc7SThomas Gleixner * package. 1458b223bc7SThomas Gleixner */ 1468b223bc7SThomas Gleixner if (bootval != ref->bootval) { 14716588f65SThomas Gleixner pr_warn(FW_BUG "TSC ADJUST differs: Reference CPU%u: %lld CPU%u: %lld\n", 1488b223bc7SThomas Gleixner refcpu, ref->bootval, cpu, bootval); 1498b223bc7SThomas Gleixner } 1508b223bc7SThomas Gleixner /* 1518b223bc7SThomas Gleixner * The TSC_ADJUST values in a package must be the same. If the boot 1528b223bc7SThomas Gleixner * value on this newly upcoming CPU differs from the adjustment 1538b223bc7SThomas Gleixner * value of the already online CPU in this package, set it to that 1548b223bc7SThomas Gleixner * adjusted value. 1558b223bc7SThomas Gleixner */ 1568b223bc7SThomas Gleixner if (bootval != ref->adjusted) { 1578b223bc7SThomas Gleixner pr_warn("TSC ADJUST synchronize: Reference CPU%u: %lld CPU%u: %lld\n", 1588b223bc7SThomas Gleixner refcpu, ref->adjusted, cpu, bootval); 1598b223bc7SThomas Gleixner cur->adjusted = ref->adjusted; 1608b223bc7SThomas Gleixner wrmsrl(MSR_IA32_TSC_ADJUST, ref->adjusted); 1618b223bc7SThomas Gleixner } 162a36f5136SThomas Gleixner /* 163a36f5136SThomas Gleixner * We have the TSCs forced to be in sync on this package. Skip sync 164a36f5136SThomas Gleixner * test: 165a36f5136SThomas Gleixner */ 166a36f5136SThomas Gleixner return true; 1678b223bc7SThomas Gleixner } 1688b223bc7SThomas Gleixner 169250c2277SThomas Gleixner /* 170250c2277SThomas Gleixner * Entry/exit counters that make sure that both CPUs 171250c2277SThomas Gleixner * run the measurement code at once: 172250c2277SThomas Gleixner */ 173148f9bb8SPaul Gortmaker static atomic_t start_count; 174148f9bb8SPaul Gortmaker static atomic_t stop_count; 175a36f5136SThomas Gleixner static atomic_t skip_test; 176cc4db268SThomas Gleixner static atomic_t test_runs; 177250c2277SThomas Gleixner 178250c2277SThomas Gleixner /* 179250c2277SThomas Gleixner * We use a raw spinlock in this exceptional case, because 180250c2277SThomas Gleixner * we want to have the fastest, inlined, non-debug version 181250c2277SThomas Gleixner * of a critical section, to be able to prove TSC time-warps: 182250c2277SThomas Gleixner */ 183148f9bb8SPaul Gortmaker static arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED; 184643bec95SIngo Molnar 185148f9bb8SPaul Gortmaker static cycles_t last_tsc; 186148f9bb8SPaul Gortmaker static cycles_t max_warp; 187148f9bb8SPaul Gortmaker static int nr_warps; 188bec8520dSThomas Gleixner static int random_warps; 189250c2277SThomas Gleixner 190250c2277SThomas Gleixner /* 191eee6946eSAndy Lutomirski * TSC-warp measurement loop running on both CPUs. This is not called 192eee6946eSAndy Lutomirski * if there is no TSC. 193250c2277SThomas Gleixner */ 19476d3b851SThomas Gleixner static cycles_t check_tsc_warp(unsigned int timeout) 195250c2277SThomas Gleixner { 19676d3b851SThomas Gleixner cycles_t start, now, prev, end, cur_max_warp = 0; 197bec8520dSThomas Gleixner int i, cur_warps = 0; 198250c2277SThomas Gleixner 199eee6946eSAndy Lutomirski start = rdtsc_ordered(); 200250c2277SThomas Gleixner /* 201b0e5c779SSuresh Siddha * The measurement runs for 'timeout' msecs: 202250c2277SThomas Gleixner */ 203b0e5c779SSuresh Siddha end = start + (cycles_t) tsc_khz * timeout; 204250c2277SThomas Gleixner now = start; 205250c2277SThomas Gleixner 206250c2277SThomas Gleixner for (i = 0; ; i++) { 207250c2277SThomas Gleixner /* 208250c2277SThomas Gleixner * We take the global lock, measure TSC, save the 209250c2277SThomas Gleixner * previous TSC that was measured (possibly on 210250c2277SThomas Gleixner * another CPU) and update the previous TSC timestamp. 211250c2277SThomas Gleixner */ 2120199c4e6SThomas Gleixner arch_spin_lock(&sync_lock); 213250c2277SThomas Gleixner prev = last_tsc; 214eee6946eSAndy Lutomirski now = rdtsc_ordered(); 215250c2277SThomas Gleixner last_tsc = now; 2160199c4e6SThomas Gleixner arch_spin_unlock(&sync_lock); 217250c2277SThomas Gleixner 218250c2277SThomas Gleixner /* 219250c2277SThomas Gleixner * Be nice every now and then (and also check whether 220df43510bSIngo Molnar * measurement is done [we also insert a 10 million 221250c2277SThomas Gleixner * loops safety exit, so we dont lock up in case the 222250c2277SThomas Gleixner * TSC readout is totally broken]): 223250c2277SThomas Gleixner */ 224250c2277SThomas Gleixner if (unlikely(!(i & 7))) { 225df43510bSIngo Molnar if (now > end || i > 10000000) 226250c2277SThomas Gleixner break; 227250c2277SThomas Gleixner cpu_relax(); 228250c2277SThomas Gleixner touch_nmi_watchdog(); 229250c2277SThomas Gleixner } 230250c2277SThomas Gleixner /* 231250c2277SThomas Gleixner * Outside the critical section we can now see whether 232250c2277SThomas Gleixner * we saw a time-warp of the TSC going backwards: 233250c2277SThomas Gleixner */ 234250c2277SThomas Gleixner if (unlikely(prev > now)) { 2350199c4e6SThomas Gleixner arch_spin_lock(&sync_lock); 236250c2277SThomas Gleixner max_warp = max(max_warp, prev - now); 23776d3b851SThomas Gleixner cur_max_warp = max_warp; 238bec8520dSThomas Gleixner /* 239bec8520dSThomas Gleixner * Check whether this bounces back and forth. Only 240bec8520dSThomas Gleixner * one CPU should observe time going backwards. 241bec8520dSThomas Gleixner */ 242bec8520dSThomas Gleixner if (cur_warps != nr_warps) 243bec8520dSThomas Gleixner random_warps++; 244250c2277SThomas Gleixner nr_warps++; 245bec8520dSThomas Gleixner cur_warps = nr_warps; 2460199c4e6SThomas Gleixner arch_spin_unlock(&sync_lock); 247250c2277SThomas Gleixner } 248ad8ca495SIngo Molnar } 249bde78a79SArjan van de Ven WARN(!(now-start), 250bde78a79SArjan van de Ven "Warning: zero tsc calibration delta: %Ld [max: %Ld]\n", 251ad8ca495SIngo Molnar now-start, end-start); 25276d3b851SThomas Gleixner return cur_max_warp; 253250c2277SThomas Gleixner } 254250c2277SThomas Gleixner 255250c2277SThomas Gleixner /* 256b0e5c779SSuresh Siddha * If the target CPU coming online doesn't have any of its core-siblings 257b0e5c779SSuresh Siddha * online, a timeout of 20msec will be used for the TSC-warp measurement 258b0e5c779SSuresh Siddha * loop. Otherwise a smaller timeout of 2msec will be used, as we have some 259b0e5c779SSuresh Siddha * information about this socket already (and this information grows as we 260b0e5c779SSuresh Siddha * have more and more logical-siblings in that socket). 261b0e5c779SSuresh Siddha * 262b0e5c779SSuresh Siddha * Ideally we should be able to skip the TSC sync check on the other 263b0e5c779SSuresh Siddha * core-siblings, if the first logical CPU in a socket passed the sync test. 264b0e5c779SSuresh Siddha * But as the TSC is per-logical CPU and can potentially be modified wrongly 265b0e5c779SSuresh Siddha * by the bios, TSC sync test for smaller duration should be able 266b0e5c779SSuresh Siddha * to catch such errors. Also this will catch the condition where all the 267b0e5c779SSuresh Siddha * cores in the socket doesn't get reset at the same time. 268b0e5c779SSuresh Siddha */ 269b0e5c779SSuresh Siddha static inline unsigned int loop_timeout(int cpu) 270b0e5c779SSuresh Siddha { 2717d79a7bdSBartosz Golaszewski return (cpumask_weight(topology_core_cpumask(cpu)) > 1) ? 2 : 20; 272b0e5c779SSuresh Siddha } 273b0e5c779SSuresh Siddha 274b0e5c779SSuresh Siddha /* 275250c2277SThomas Gleixner * Source CPU calls into this - it waits for the freshly booted 276250c2277SThomas Gleixner * target CPU to arrive and then starts the measurement: 277250c2277SThomas Gleixner */ 278148f9bb8SPaul Gortmaker void check_tsc_sync_source(int cpu) 279250c2277SThomas Gleixner { 280250c2277SThomas Gleixner int cpus = 2; 281250c2277SThomas Gleixner 282250c2277SThomas Gleixner /* 283250c2277SThomas Gleixner * No need to check if we already know that the TSC is not 284eee6946eSAndy Lutomirski * synchronized or if we have no TSC. 285250c2277SThomas Gleixner */ 286250c2277SThomas Gleixner if (unsynchronized_tsc()) 287250c2277SThomas Gleixner return; 288250c2277SThomas Gleixner 28928a00184SSuresh Siddha if (tsc_clocksource_reliable) { 2909b3660a5SMike Travis if (cpu == (nr_cpu_ids-1) || system_state != SYSTEM_BOOTING) 2919b3660a5SMike Travis pr_info( 2929b3660a5SMike Travis "Skipped synchronization checks as TSC is reliable.\n"); 293eca0cd02SAlok Kataria return; 294eca0cd02SAlok Kataria } 295eca0cd02SAlok Kataria 296250c2277SThomas Gleixner /* 297cc4db268SThomas Gleixner * Set the maximum number of test runs to 298cc4db268SThomas Gleixner * 1 if the CPU does not provide the TSC_ADJUST MSR 299cc4db268SThomas Gleixner * 3 if the MSR is available, so the target can try to adjust 300cc4db268SThomas Gleixner */ 301cc4db268SThomas Gleixner if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) 302cc4db268SThomas Gleixner atomic_set(&test_runs, 1); 303cc4db268SThomas Gleixner else 304cc4db268SThomas Gleixner atomic_set(&test_runs, 3); 305cc4db268SThomas Gleixner retry: 306cc4db268SThomas Gleixner /* 307a36f5136SThomas Gleixner * Wait for the target to start or to skip the test: 308250c2277SThomas Gleixner */ 309a36f5136SThomas Gleixner while (atomic_read(&start_count) != cpus - 1) { 310a36f5136SThomas Gleixner if (atomic_read(&skip_test) > 0) { 311a36f5136SThomas Gleixner atomic_set(&skip_test, 0); 312a36f5136SThomas Gleixner return; 313a36f5136SThomas Gleixner } 314250c2277SThomas Gleixner cpu_relax(); 315a36f5136SThomas Gleixner } 316a36f5136SThomas Gleixner 317250c2277SThomas Gleixner /* 318250c2277SThomas Gleixner * Trigger the target to continue into the measurement too: 319250c2277SThomas Gleixner */ 320250c2277SThomas Gleixner atomic_inc(&start_count); 321250c2277SThomas Gleixner 322b0e5c779SSuresh Siddha check_tsc_warp(loop_timeout(cpu)); 323250c2277SThomas Gleixner 324250c2277SThomas Gleixner while (atomic_read(&stop_count) != cpus-1) 325250c2277SThomas Gleixner cpu_relax(); 326250c2277SThomas Gleixner 327cc4db268SThomas Gleixner /* 328cc4db268SThomas Gleixner * If the test was successful set the number of runs to zero and 329cc4db268SThomas Gleixner * stop. If not, decrement the number of runs an check if we can 330cc4db268SThomas Gleixner * retry. In case of random warps no retry is attempted. 331cc4db268SThomas Gleixner */ 332cc4db268SThomas Gleixner if (!nr_warps) { 333cc4db268SThomas Gleixner atomic_set(&test_runs, 0); 334cc4db268SThomas Gleixner 335cc4db268SThomas Gleixner pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n", 336cc4db268SThomas Gleixner smp_processor_id(), cpu); 337cc4db268SThomas Gleixner 338cc4db268SThomas Gleixner } else if (atomic_dec_and_test(&test_runs) || random_warps) { 339cc4db268SThomas Gleixner /* Force it to 0 if random warps brought us here */ 340cc4db268SThomas Gleixner atomic_set(&test_runs, 0); 341cc4db268SThomas Gleixner 3429b3660a5SMike Travis pr_warning("TSC synchronization [CPU#%d -> CPU#%d]:\n", 3439b3660a5SMike Travis smp_processor_id(), cpu); 344643bec95SIngo Molnar pr_warning("Measured %Ld cycles TSC warp between CPUs, " 345250c2277SThomas Gleixner "turning off TSC clock.\n", max_warp); 346bec8520dSThomas Gleixner if (random_warps) 347bec8520dSThomas Gleixner pr_warning("TSC warped randomly between CPUs\n"); 348250c2277SThomas Gleixner mark_tsc_unstable("check_tsc_sync_source failed"); 349250c2277SThomas Gleixner } 350250c2277SThomas Gleixner 351250c2277SThomas Gleixner /* 3524c6b8b4dSMike Galbraith * Reset it - just in case we boot another CPU later: 3534c6b8b4dSMike Galbraith */ 3544c6b8b4dSMike Galbraith atomic_set(&start_count, 0); 355bec8520dSThomas Gleixner random_warps = 0; 3564c6b8b4dSMike Galbraith nr_warps = 0; 3574c6b8b4dSMike Galbraith max_warp = 0; 3584c6b8b4dSMike Galbraith last_tsc = 0; 3594c6b8b4dSMike Galbraith 3604c6b8b4dSMike Galbraith /* 361250c2277SThomas Gleixner * Let the target continue with the bootup: 362250c2277SThomas Gleixner */ 363250c2277SThomas Gleixner atomic_inc(&stop_count); 364cc4db268SThomas Gleixner 365cc4db268SThomas Gleixner /* 366cc4db268SThomas Gleixner * Retry, if there is a chance to do so. 367cc4db268SThomas Gleixner */ 368cc4db268SThomas Gleixner if (atomic_read(&test_runs) > 0) 369cc4db268SThomas Gleixner goto retry; 370250c2277SThomas Gleixner } 371250c2277SThomas Gleixner 372250c2277SThomas Gleixner /* 373250c2277SThomas Gleixner * Freshly booted CPUs call into this: 374250c2277SThomas Gleixner */ 375148f9bb8SPaul Gortmaker void check_tsc_sync_target(void) 376250c2277SThomas Gleixner { 377cc4db268SThomas Gleixner struct tsc_adjust *cur = this_cpu_ptr(&tsc_adjust); 378cc4db268SThomas Gleixner unsigned int cpu = smp_processor_id(); 379cc4db268SThomas Gleixner cycles_t cur_max_warp, gbl_max_warp; 380250c2277SThomas Gleixner int cpus = 2; 381250c2277SThomas Gleixner 382eee6946eSAndy Lutomirski /* Also aborts if there is no TSC. */ 38328a00184SSuresh Siddha if (unsynchronized_tsc() || tsc_clocksource_reliable) 384250c2277SThomas Gleixner return; 385250c2277SThomas Gleixner 386a36f5136SThomas Gleixner /* 387a36f5136SThomas Gleixner * Store, verify and sanitize the TSC adjust register. If 388a36f5136SThomas Gleixner * successful skip the test. 389a36f5136SThomas Gleixner */ 3905bae1562SThomas Gleixner if (tsc_store_and_check_tsc_adjust(false)) { 391a36f5136SThomas Gleixner atomic_inc(&skip_test); 392a36f5136SThomas Gleixner return; 393a36f5136SThomas Gleixner } 3948b223bc7SThomas Gleixner 395cc4db268SThomas Gleixner retry: 396250c2277SThomas Gleixner /* 397250c2277SThomas Gleixner * Register this CPU's participation and wait for the 398250c2277SThomas Gleixner * source CPU to start the measurement: 399250c2277SThomas Gleixner */ 400250c2277SThomas Gleixner atomic_inc(&start_count); 401250c2277SThomas Gleixner while (atomic_read(&start_count) != cpus) 402250c2277SThomas Gleixner cpu_relax(); 403250c2277SThomas Gleixner 404cc4db268SThomas Gleixner cur_max_warp = check_tsc_warp(loop_timeout(cpu)); 405cc4db268SThomas Gleixner 406cc4db268SThomas Gleixner /* 407cc4db268SThomas Gleixner * Store the maximum observed warp value for a potential retry: 408cc4db268SThomas Gleixner */ 409cc4db268SThomas Gleixner gbl_max_warp = max_warp; 410250c2277SThomas Gleixner 411250c2277SThomas Gleixner /* 412250c2277SThomas Gleixner * Ok, we are done: 413250c2277SThomas Gleixner */ 414250c2277SThomas Gleixner atomic_inc(&stop_count); 415250c2277SThomas Gleixner 416250c2277SThomas Gleixner /* 417250c2277SThomas Gleixner * Wait for the source CPU to print stuff: 418250c2277SThomas Gleixner */ 419250c2277SThomas Gleixner while (atomic_read(&stop_count) != cpus) 420250c2277SThomas Gleixner cpu_relax(); 4214c5e3c63SThomas Gleixner 4224c5e3c63SThomas Gleixner /* 4234c5e3c63SThomas Gleixner * Reset it for the next sync test: 4244c5e3c63SThomas Gleixner */ 4254c5e3c63SThomas Gleixner atomic_set(&stop_count, 0); 426cc4db268SThomas Gleixner 427cc4db268SThomas Gleixner /* 428cc4db268SThomas Gleixner * Check the number of remaining test runs. If not zero, the test 429cc4db268SThomas Gleixner * failed and a retry with adjusted TSC is possible. If zero the 430cc4db268SThomas Gleixner * test was either successful or failed terminally. 431cc4db268SThomas Gleixner */ 432cc4db268SThomas Gleixner if (!atomic_read(&test_runs)) 433cc4db268SThomas Gleixner return; 434cc4db268SThomas Gleixner 435cc4db268SThomas Gleixner /* 436cc4db268SThomas Gleixner * If the warp value of this CPU is 0, then the other CPU 437cc4db268SThomas Gleixner * observed time going backwards so this TSC was ahead and 438cc4db268SThomas Gleixner * needs to move backwards. 439cc4db268SThomas Gleixner */ 440cc4db268SThomas Gleixner if (!cur_max_warp) 441cc4db268SThomas Gleixner cur_max_warp = -gbl_max_warp; 442cc4db268SThomas Gleixner 443cc4db268SThomas Gleixner /* 444cc4db268SThomas Gleixner * Add the result to the previous adjustment value. 445cc4db268SThomas Gleixner * 446cc4db268SThomas Gleixner * The adjustement value is slightly off by the overhead of the 447cc4db268SThomas Gleixner * sync mechanism (observed values are ~200 TSC cycles), but this 448cc4db268SThomas Gleixner * really depends on CPU, node distance and frequency. So 449cc4db268SThomas Gleixner * compensating for this is hard to get right. Experiments show 450cc4db268SThomas Gleixner * that the warp is not longer detectable when the observed warp 451cc4db268SThomas Gleixner * value is used. In the worst case the adjustment needs to go 452cc4db268SThomas Gleixner * through a 3rd run for fine tuning. 453cc4db268SThomas Gleixner */ 454cc4db268SThomas Gleixner cur->adjusted += cur_max_warp; 4558c9b9d87SThomas Gleixner 4568c9b9d87SThomas Gleixner /* 4578c9b9d87SThomas Gleixner * TSC deadline timer stops working or creates an interrupt storm 4588c9b9d87SThomas Gleixner * with adjust values < 0 and > x07ffffff. 4598c9b9d87SThomas Gleixner * 4608c9b9d87SThomas Gleixner * To allow adjust values > 0x7FFFFFFF we need to disable the 4618c9b9d87SThomas Gleixner * deadline timer and use the local APIC timer, but that requires 4628c9b9d87SThomas Gleixner * more intrusive changes and we do not have any useful information 4638c9b9d87SThomas Gleixner * from Intel about the underlying HW wreckage yet. 4648c9b9d87SThomas Gleixner */ 4655bae1562SThomas Gleixner if (cur->adjusted < 0) 4665bae1562SThomas Gleixner cur->adjusted = 0; 4678c9b9d87SThomas Gleixner if (cur->adjusted > 0x7FFFFFFF) 4688c9b9d87SThomas Gleixner cur->adjusted = 0x7FFFFFFF; 469cc4db268SThomas Gleixner 470cc4db268SThomas Gleixner pr_warn("TSC ADJUST compensate: CPU%u observed %lld warp. Adjust: %lld\n", 471cc4db268SThomas Gleixner cpu, cur_max_warp, cur->adjusted); 472cc4db268SThomas Gleixner 473cc4db268SThomas Gleixner wrmsrl(MSR_IA32_TSC_ADJUST, cur->adjusted); 474cc4db268SThomas Gleixner goto retry; 475cc4db268SThomas Gleixner 476250c2277SThomas Gleixner } 4778b223bc7SThomas Gleixner 4788b223bc7SThomas Gleixner #endif /* CONFIG_SMP */ 479