1250c2277SThomas Gleixner /* 2835c34a1SDave Jones * check TSC synchronization. 3250c2277SThomas Gleixner * 4250c2277SThomas Gleixner * Copyright (C) 2006, Red Hat, Inc., Ingo Molnar 5250c2277SThomas Gleixner * 6250c2277SThomas Gleixner * We check whether all boot CPUs have their TSC's synchronized, 7250c2277SThomas Gleixner * print a warning if not and turn off the TSC clock-source. 8250c2277SThomas Gleixner * 9250c2277SThomas Gleixner * The warp-check is point-to-point between two CPUs, the CPU 10250c2277SThomas Gleixner * initiating the bootup is the 'source CPU', the freshly booting 11250c2277SThomas Gleixner * CPU is the 'target CPU'. 12250c2277SThomas Gleixner * 13250c2277SThomas Gleixner * Only two CPUs may participate - they can enter in any order. 14250c2277SThomas Gleixner * ( The serial nature of the boot logic and the CPU hotplug lock 15250c2277SThomas Gleixner * protects against more than 2 CPUs entering this code. ) 16250c2277SThomas Gleixner */ 178b223bc7SThomas Gleixner #include <linux/topology.h> 18250c2277SThomas Gleixner #include <linux/spinlock.h> 19250c2277SThomas Gleixner #include <linux/kernel.h> 20250c2277SThomas Gleixner #include <linux/smp.h> 21250c2277SThomas Gleixner #include <linux/nmi.h> 22250c2277SThomas Gleixner #include <asm/tsc.h> 23250c2277SThomas Gleixner 248b223bc7SThomas Gleixner struct tsc_adjust { 258b223bc7SThomas Gleixner s64 bootval; 268b223bc7SThomas Gleixner s64 adjusted; 271d0095feSThomas Gleixner unsigned long nextcheck; 281d0095feSThomas Gleixner bool warned; 298b223bc7SThomas Gleixner }; 308b223bc7SThomas Gleixner 318b223bc7SThomas Gleixner static DEFINE_PER_CPU(struct tsc_adjust, tsc_adjust); 328b223bc7SThomas Gleixner 331d0095feSThomas Gleixner void tsc_verify_tsc_adjust(void) 341d0095feSThomas Gleixner { 351d0095feSThomas Gleixner struct tsc_adjust *adj = this_cpu_ptr(&tsc_adjust); 361d0095feSThomas Gleixner s64 curval; 371d0095feSThomas Gleixner 381d0095feSThomas Gleixner if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) 391d0095feSThomas Gleixner return; 401d0095feSThomas Gleixner 411d0095feSThomas Gleixner /* Rate limit the MSR check */ 421d0095feSThomas Gleixner if (time_before(jiffies, adj->nextcheck)) 431d0095feSThomas Gleixner return; 441d0095feSThomas Gleixner 451d0095feSThomas Gleixner adj->nextcheck = jiffies + HZ; 461d0095feSThomas Gleixner 471d0095feSThomas Gleixner rdmsrl(MSR_IA32_TSC_ADJUST, curval); 481d0095feSThomas Gleixner if (adj->adjusted == curval) 491d0095feSThomas Gleixner return; 501d0095feSThomas Gleixner 511d0095feSThomas Gleixner /* Restore the original value */ 521d0095feSThomas Gleixner wrmsrl(MSR_IA32_TSC_ADJUST, adj->adjusted); 531d0095feSThomas Gleixner 541d0095feSThomas Gleixner if (!adj->warned) { 551d0095feSThomas Gleixner pr_warn(FW_BUG "TSC ADJUST differs: CPU%u %lld --> %lld. Restoring\n", 561d0095feSThomas Gleixner smp_processor_id(), adj->adjusted, curval); 571d0095feSThomas Gleixner adj->warned = true; 581d0095feSThomas Gleixner } 591d0095feSThomas Gleixner } 601d0095feSThomas Gleixner 618b223bc7SThomas Gleixner #ifndef CONFIG_SMP 62a36f5136SThomas Gleixner bool __init tsc_store_and_check_tsc_adjust(void) 638b223bc7SThomas Gleixner { 648b223bc7SThomas Gleixner struct tsc_adjust *ref, *cur = this_cpu_ptr(&tsc_adjust); 658b223bc7SThomas Gleixner s64 bootval; 668b223bc7SThomas Gleixner 678b223bc7SThomas Gleixner if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) 68a36f5136SThomas Gleixner return false; 698b223bc7SThomas Gleixner 708b223bc7SThomas Gleixner rdmsrl(MSR_IA32_TSC_ADJUST, bootval); 718b223bc7SThomas Gleixner cur->bootval = bootval; 728b223bc7SThomas Gleixner cur->adjusted = bootval; 731d0095feSThomas Gleixner cur->nextcheck = jiffies + HZ; 748b223bc7SThomas Gleixner pr_info("TSC ADJUST: Boot CPU0: %lld\n", bootval); 75a36f5136SThomas Gleixner return false; 768b223bc7SThomas Gleixner } 778b223bc7SThomas Gleixner 788b223bc7SThomas Gleixner #else /* !CONFIG_SMP */ 798b223bc7SThomas Gleixner 808b223bc7SThomas Gleixner /* 818b223bc7SThomas Gleixner * Store and check the TSC ADJUST MSR if available 828b223bc7SThomas Gleixner */ 83a36f5136SThomas Gleixner bool tsc_store_and_check_tsc_adjust(void) 848b223bc7SThomas Gleixner { 858b223bc7SThomas Gleixner struct tsc_adjust *ref, *cur = this_cpu_ptr(&tsc_adjust); 868b223bc7SThomas Gleixner unsigned int refcpu, cpu = smp_processor_id(); 878b223bc7SThomas Gleixner s64 bootval; 888b223bc7SThomas Gleixner 898b223bc7SThomas Gleixner if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) 90a36f5136SThomas Gleixner return false; 918b223bc7SThomas Gleixner 928b223bc7SThomas Gleixner rdmsrl(MSR_IA32_TSC_ADJUST, bootval); 938b223bc7SThomas Gleixner cur->bootval = bootval; 941d0095feSThomas Gleixner cur->nextcheck = jiffies + HZ; 951d0095feSThomas Gleixner cur->warned = false; 968b223bc7SThomas Gleixner 978b223bc7SThomas Gleixner /* 988b223bc7SThomas Gleixner * Check whether this CPU is the first in a package to come up. In 998b223bc7SThomas Gleixner * this case do not check the boot value against another package 1008b223bc7SThomas Gleixner * because the package might have been physically hotplugged, where 1018b223bc7SThomas Gleixner * TSC_ADJUST is expected to be different. 1028b223bc7SThomas Gleixner */ 1038b223bc7SThomas Gleixner refcpu = cpumask_any_but(topology_core_cpumask(cpu), cpu); 1048b223bc7SThomas Gleixner 1058b223bc7SThomas Gleixner if (refcpu >= nr_cpu_ids) { 1068b223bc7SThomas Gleixner /* 1078b223bc7SThomas Gleixner * First online CPU in a package stores the boot value in 1088b223bc7SThomas Gleixner * the adjustment value. This value might change later via 1098b223bc7SThomas Gleixner * the sync mechanism. If that fails we still can yell 1108b223bc7SThomas Gleixner * about boot values not being consistent. 1118b223bc7SThomas Gleixner */ 1128b223bc7SThomas Gleixner cur->adjusted = bootval; 1138b223bc7SThomas Gleixner pr_info_once("TSC ADJUST: Boot CPU%u: %lld\n", cpu, bootval); 114a36f5136SThomas Gleixner return false; 1158b223bc7SThomas Gleixner } 1168b223bc7SThomas Gleixner 1178b223bc7SThomas Gleixner ref = per_cpu_ptr(&tsc_adjust, refcpu); 1188b223bc7SThomas Gleixner /* 1198b223bc7SThomas Gleixner * Compare the boot value and complain if it differs in the 1208b223bc7SThomas Gleixner * package. 1218b223bc7SThomas Gleixner */ 1228b223bc7SThomas Gleixner if (bootval != ref->bootval) { 1238b223bc7SThomas Gleixner pr_warn("TSC ADJUST differs: Reference CPU%u: %lld CPU%u: %lld\n", 1248b223bc7SThomas Gleixner refcpu, ref->bootval, cpu, bootval); 1258b223bc7SThomas Gleixner } 1268b223bc7SThomas Gleixner /* 1278b223bc7SThomas Gleixner * The TSC_ADJUST values in a package must be the same. If the boot 1288b223bc7SThomas Gleixner * value on this newly upcoming CPU differs from the adjustment 1298b223bc7SThomas Gleixner * value of the already online CPU in this package, set it to that 1308b223bc7SThomas Gleixner * adjusted value. 1318b223bc7SThomas Gleixner */ 1328b223bc7SThomas Gleixner if (bootval != ref->adjusted) { 1338b223bc7SThomas Gleixner pr_warn("TSC ADJUST synchronize: Reference CPU%u: %lld CPU%u: %lld\n", 1348b223bc7SThomas Gleixner refcpu, ref->adjusted, cpu, bootval); 1358b223bc7SThomas Gleixner cur->adjusted = ref->adjusted; 1368b223bc7SThomas Gleixner wrmsrl(MSR_IA32_TSC_ADJUST, ref->adjusted); 1378b223bc7SThomas Gleixner } 138a36f5136SThomas Gleixner /* 139a36f5136SThomas Gleixner * We have the TSCs forced to be in sync on this package. Skip sync 140a36f5136SThomas Gleixner * test: 141a36f5136SThomas Gleixner */ 142a36f5136SThomas Gleixner return true; 1438b223bc7SThomas Gleixner } 1448b223bc7SThomas Gleixner 145250c2277SThomas Gleixner /* 146250c2277SThomas Gleixner * Entry/exit counters that make sure that both CPUs 147250c2277SThomas Gleixner * run the measurement code at once: 148250c2277SThomas Gleixner */ 149148f9bb8SPaul Gortmaker static atomic_t start_count; 150148f9bb8SPaul Gortmaker static atomic_t stop_count; 151a36f5136SThomas Gleixner static atomic_t skip_test; 152cc4db268SThomas Gleixner static atomic_t test_runs; 153250c2277SThomas Gleixner 154250c2277SThomas Gleixner /* 155250c2277SThomas Gleixner * We use a raw spinlock in this exceptional case, because 156250c2277SThomas Gleixner * we want to have the fastest, inlined, non-debug version 157250c2277SThomas Gleixner * of a critical section, to be able to prove TSC time-warps: 158250c2277SThomas Gleixner */ 159148f9bb8SPaul Gortmaker static arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED; 160643bec95SIngo Molnar 161148f9bb8SPaul Gortmaker static cycles_t last_tsc; 162148f9bb8SPaul Gortmaker static cycles_t max_warp; 163148f9bb8SPaul Gortmaker static int nr_warps; 164bec8520dSThomas Gleixner static int random_warps; 165250c2277SThomas Gleixner 166250c2277SThomas Gleixner /* 167eee6946eSAndy Lutomirski * TSC-warp measurement loop running on both CPUs. This is not called 168eee6946eSAndy Lutomirski * if there is no TSC. 169250c2277SThomas Gleixner */ 17076d3b851SThomas Gleixner static cycles_t check_tsc_warp(unsigned int timeout) 171250c2277SThomas Gleixner { 17276d3b851SThomas Gleixner cycles_t start, now, prev, end, cur_max_warp = 0; 173bec8520dSThomas Gleixner int i, cur_warps = 0; 174250c2277SThomas Gleixner 175eee6946eSAndy Lutomirski start = rdtsc_ordered(); 176250c2277SThomas Gleixner /* 177b0e5c779SSuresh Siddha * The measurement runs for 'timeout' msecs: 178250c2277SThomas Gleixner */ 179b0e5c779SSuresh Siddha end = start + (cycles_t) tsc_khz * timeout; 180250c2277SThomas Gleixner now = start; 181250c2277SThomas Gleixner 182250c2277SThomas Gleixner for (i = 0; ; i++) { 183250c2277SThomas Gleixner /* 184250c2277SThomas Gleixner * We take the global lock, measure TSC, save the 185250c2277SThomas Gleixner * previous TSC that was measured (possibly on 186250c2277SThomas Gleixner * another CPU) and update the previous TSC timestamp. 187250c2277SThomas Gleixner */ 1880199c4e6SThomas Gleixner arch_spin_lock(&sync_lock); 189250c2277SThomas Gleixner prev = last_tsc; 190eee6946eSAndy Lutomirski now = rdtsc_ordered(); 191250c2277SThomas Gleixner last_tsc = now; 1920199c4e6SThomas Gleixner arch_spin_unlock(&sync_lock); 193250c2277SThomas Gleixner 194250c2277SThomas Gleixner /* 195250c2277SThomas Gleixner * Be nice every now and then (and also check whether 196df43510bSIngo Molnar * measurement is done [we also insert a 10 million 197250c2277SThomas Gleixner * loops safety exit, so we dont lock up in case the 198250c2277SThomas Gleixner * TSC readout is totally broken]): 199250c2277SThomas Gleixner */ 200250c2277SThomas Gleixner if (unlikely(!(i & 7))) { 201df43510bSIngo Molnar if (now > end || i > 10000000) 202250c2277SThomas Gleixner break; 203250c2277SThomas Gleixner cpu_relax(); 204250c2277SThomas Gleixner touch_nmi_watchdog(); 205250c2277SThomas Gleixner } 206250c2277SThomas Gleixner /* 207250c2277SThomas Gleixner * Outside the critical section we can now see whether 208250c2277SThomas Gleixner * we saw a time-warp of the TSC going backwards: 209250c2277SThomas Gleixner */ 210250c2277SThomas Gleixner if (unlikely(prev > now)) { 2110199c4e6SThomas Gleixner arch_spin_lock(&sync_lock); 212250c2277SThomas Gleixner max_warp = max(max_warp, prev - now); 21376d3b851SThomas Gleixner cur_max_warp = max_warp; 214bec8520dSThomas Gleixner /* 215bec8520dSThomas Gleixner * Check whether this bounces back and forth. Only 216bec8520dSThomas Gleixner * one CPU should observe time going backwards. 217bec8520dSThomas Gleixner */ 218bec8520dSThomas Gleixner if (cur_warps != nr_warps) 219bec8520dSThomas Gleixner random_warps++; 220250c2277SThomas Gleixner nr_warps++; 221bec8520dSThomas Gleixner cur_warps = nr_warps; 2220199c4e6SThomas Gleixner arch_spin_unlock(&sync_lock); 223250c2277SThomas Gleixner } 224ad8ca495SIngo Molnar } 225bde78a79SArjan van de Ven WARN(!(now-start), 226bde78a79SArjan van de Ven "Warning: zero tsc calibration delta: %Ld [max: %Ld]\n", 227ad8ca495SIngo Molnar now-start, end-start); 22876d3b851SThomas Gleixner return cur_max_warp; 229250c2277SThomas Gleixner } 230250c2277SThomas Gleixner 231250c2277SThomas Gleixner /* 232b0e5c779SSuresh Siddha * If the target CPU coming online doesn't have any of its core-siblings 233b0e5c779SSuresh Siddha * online, a timeout of 20msec will be used for the TSC-warp measurement 234b0e5c779SSuresh Siddha * loop. Otherwise a smaller timeout of 2msec will be used, as we have some 235b0e5c779SSuresh Siddha * information about this socket already (and this information grows as we 236b0e5c779SSuresh Siddha * have more and more logical-siblings in that socket). 237b0e5c779SSuresh Siddha * 238b0e5c779SSuresh Siddha * Ideally we should be able to skip the TSC sync check on the other 239b0e5c779SSuresh Siddha * core-siblings, if the first logical CPU in a socket passed the sync test. 240b0e5c779SSuresh Siddha * But as the TSC is per-logical CPU and can potentially be modified wrongly 241b0e5c779SSuresh Siddha * by the bios, TSC sync test for smaller duration should be able 242b0e5c779SSuresh Siddha * to catch such errors. Also this will catch the condition where all the 243b0e5c779SSuresh Siddha * cores in the socket doesn't get reset at the same time. 244b0e5c779SSuresh Siddha */ 245b0e5c779SSuresh Siddha static inline unsigned int loop_timeout(int cpu) 246b0e5c779SSuresh Siddha { 2477d79a7bdSBartosz Golaszewski return (cpumask_weight(topology_core_cpumask(cpu)) > 1) ? 2 : 20; 248b0e5c779SSuresh Siddha } 249b0e5c779SSuresh Siddha 250b0e5c779SSuresh Siddha /* 251250c2277SThomas Gleixner * Source CPU calls into this - it waits for the freshly booted 252250c2277SThomas Gleixner * target CPU to arrive and then starts the measurement: 253250c2277SThomas Gleixner */ 254148f9bb8SPaul Gortmaker void check_tsc_sync_source(int cpu) 255250c2277SThomas Gleixner { 256250c2277SThomas Gleixner int cpus = 2; 257250c2277SThomas Gleixner 258250c2277SThomas Gleixner /* 259250c2277SThomas Gleixner * No need to check if we already know that the TSC is not 260eee6946eSAndy Lutomirski * synchronized or if we have no TSC. 261250c2277SThomas Gleixner */ 262250c2277SThomas Gleixner if (unsynchronized_tsc()) 263250c2277SThomas Gleixner return; 264250c2277SThomas Gleixner 26528a00184SSuresh Siddha if (tsc_clocksource_reliable) { 2669b3660a5SMike Travis if (cpu == (nr_cpu_ids-1) || system_state != SYSTEM_BOOTING) 2679b3660a5SMike Travis pr_info( 2689b3660a5SMike Travis "Skipped synchronization checks as TSC is reliable.\n"); 269eca0cd02SAlok Kataria return; 270eca0cd02SAlok Kataria } 271eca0cd02SAlok Kataria 272250c2277SThomas Gleixner /* 273cc4db268SThomas Gleixner * Set the maximum number of test runs to 274cc4db268SThomas Gleixner * 1 if the CPU does not provide the TSC_ADJUST MSR 275cc4db268SThomas Gleixner * 3 if the MSR is available, so the target can try to adjust 276cc4db268SThomas Gleixner */ 277cc4db268SThomas Gleixner if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) 278cc4db268SThomas Gleixner atomic_set(&test_runs, 1); 279cc4db268SThomas Gleixner else 280cc4db268SThomas Gleixner atomic_set(&test_runs, 3); 281cc4db268SThomas Gleixner retry: 282cc4db268SThomas Gleixner /* 283a36f5136SThomas Gleixner * Wait for the target to start or to skip the test: 284250c2277SThomas Gleixner */ 285a36f5136SThomas Gleixner while (atomic_read(&start_count) != cpus - 1) { 286a36f5136SThomas Gleixner if (atomic_read(&skip_test) > 0) { 287a36f5136SThomas Gleixner atomic_set(&skip_test, 0); 288a36f5136SThomas Gleixner return; 289a36f5136SThomas Gleixner } 290250c2277SThomas Gleixner cpu_relax(); 291a36f5136SThomas Gleixner } 292a36f5136SThomas Gleixner 293250c2277SThomas Gleixner /* 294250c2277SThomas Gleixner * Trigger the target to continue into the measurement too: 295250c2277SThomas Gleixner */ 296250c2277SThomas Gleixner atomic_inc(&start_count); 297250c2277SThomas Gleixner 298b0e5c779SSuresh Siddha check_tsc_warp(loop_timeout(cpu)); 299250c2277SThomas Gleixner 300250c2277SThomas Gleixner while (atomic_read(&stop_count) != cpus-1) 301250c2277SThomas Gleixner cpu_relax(); 302250c2277SThomas Gleixner 303cc4db268SThomas Gleixner /* 304cc4db268SThomas Gleixner * If the test was successful set the number of runs to zero and 305cc4db268SThomas Gleixner * stop. If not, decrement the number of runs an check if we can 306cc4db268SThomas Gleixner * retry. In case of random warps no retry is attempted. 307cc4db268SThomas Gleixner */ 308cc4db268SThomas Gleixner if (!nr_warps) { 309cc4db268SThomas Gleixner atomic_set(&test_runs, 0); 310cc4db268SThomas Gleixner 311cc4db268SThomas Gleixner pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n", 312cc4db268SThomas Gleixner smp_processor_id(), cpu); 313cc4db268SThomas Gleixner 314cc4db268SThomas Gleixner } else if (atomic_dec_and_test(&test_runs) || random_warps) { 315cc4db268SThomas Gleixner /* Force it to 0 if random warps brought us here */ 316cc4db268SThomas Gleixner atomic_set(&test_runs, 0); 317cc4db268SThomas Gleixner 3189b3660a5SMike Travis pr_warning("TSC synchronization [CPU#%d -> CPU#%d]:\n", 3199b3660a5SMike Travis smp_processor_id(), cpu); 320643bec95SIngo Molnar pr_warning("Measured %Ld cycles TSC warp between CPUs, " 321250c2277SThomas Gleixner "turning off TSC clock.\n", max_warp); 322bec8520dSThomas Gleixner if (random_warps) 323bec8520dSThomas Gleixner pr_warning("TSC warped randomly between CPUs\n"); 324250c2277SThomas Gleixner mark_tsc_unstable("check_tsc_sync_source failed"); 325250c2277SThomas Gleixner } 326250c2277SThomas Gleixner 327250c2277SThomas Gleixner /* 3284c6b8b4dSMike Galbraith * Reset it - just in case we boot another CPU later: 3294c6b8b4dSMike Galbraith */ 3304c6b8b4dSMike Galbraith atomic_set(&start_count, 0); 331bec8520dSThomas Gleixner random_warps = 0; 3324c6b8b4dSMike Galbraith nr_warps = 0; 3334c6b8b4dSMike Galbraith max_warp = 0; 3344c6b8b4dSMike Galbraith last_tsc = 0; 3354c6b8b4dSMike Galbraith 3364c6b8b4dSMike Galbraith /* 337250c2277SThomas Gleixner * Let the target continue with the bootup: 338250c2277SThomas Gleixner */ 339250c2277SThomas Gleixner atomic_inc(&stop_count); 340cc4db268SThomas Gleixner 341cc4db268SThomas Gleixner /* 342cc4db268SThomas Gleixner * Retry, if there is a chance to do so. 343cc4db268SThomas Gleixner */ 344cc4db268SThomas Gleixner if (atomic_read(&test_runs) > 0) 345cc4db268SThomas Gleixner goto retry; 346250c2277SThomas Gleixner } 347250c2277SThomas Gleixner 348250c2277SThomas Gleixner /* 349250c2277SThomas Gleixner * Freshly booted CPUs call into this: 350250c2277SThomas Gleixner */ 351148f9bb8SPaul Gortmaker void check_tsc_sync_target(void) 352250c2277SThomas Gleixner { 353cc4db268SThomas Gleixner struct tsc_adjust *cur = this_cpu_ptr(&tsc_adjust); 354cc4db268SThomas Gleixner unsigned int cpu = smp_processor_id(); 355cc4db268SThomas Gleixner cycles_t cur_max_warp, gbl_max_warp; 356250c2277SThomas Gleixner int cpus = 2; 357250c2277SThomas Gleixner 358eee6946eSAndy Lutomirski /* Also aborts if there is no TSC. */ 35928a00184SSuresh Siddha if (unsynchronized_tsc() || tsc_clocksource_reliable) 360250c2277SThomas Gleixner return; 361250c2277SThomas Gleixner 362a36f5136SThomas Gleixner /* 363a36f5136SThomas Gleixner * Store, verify and sanitize the TSC adjust register. If 364a36f5136SThomas Gleixner * successful skip the test. 365a36f5136SThomas Gleixner */ 366a36f5136SThomas Gleixner if (tsc_store_and_check_tsc_adjust()) { 367a36f5136SThomas Gleixner atomic_inc(&skip_test); 368a36f5136SThomas Gleixner return; 369a36f5136SThomas Gleixner } 3708b223bc7SThomas Gleixner 371cc4db268SThomas Gleixner retry: 372250c2277SThomas Gleixner /* 373250c2277SThomas Gleixner * Register this CPU's participation and wait for the 374250c2277SThomas Gleixner * source CPU to start the measurement: 375250c2277SThomas Gleixner */ 376250c2277SThomas Gleixner atomic_inc(&start_count); 377250c2277SThomas Gleixner while (atomic_read(&start_count) != cpus) 378250c2277SThomas Gleixner cpu_relax(); 379250c2277SThomas Gleixner 380cc4db268SThomas Gleixner cur_max_warp = check_tsc_warp(loop_timeout(cpu)); 381cc4db268SThomas Gleixner 382cc4db268SThomas Gleixner /* 383cc4db268SThomas Gleixner * Store the maximum observed warp value for a potential retry: 384cc4db268SThomas Gleixner */ 385cc4db268SThomas Gleixner gbl_max_warp = max_warp; 386250c2277SThomas Gleixner 387250c2277SThomas Gleixner /* 388250c2277SThomas Gleixner * Ok, we are done: 389250c2277SThomas Gleixner */ 390250c2277SThomas Gleixner atomic_inc(&stop_count); 391250c2277SThomas Gleixner 392250c2277SThomas Gleixner /* 393250c2277SThomas Gleixner * Wait for the source CPU to print stuff: 394250c2277SThomas Gleixner */ 395250c2277SThomas Gleixner while (atomic_read(&stop_count) != cpus) 396250c2277SThomas Gleixner cpu_relax(); 3974c5e3c63SThomas Gleixner 3984c5e3c63SThomas Gleixner /* 3994c5e3c63SThomas Gleixner * Reset it for the next sync test: 4004c5e3c63SThomas Gleixner */ 4014c5e3c63SThomas Gleixner atomic_set(&stop_count, 0); 402cc4db268SThomas Gleixner 403cc4db268SThomas Gleixner /* 404cc4db268SThomas Gleixner * Check the number of remaining test runs. If not zero, the test 405cc4db268SThomas Gleixner * failed and a retry with adjusted TSC is possible. If zero the 406cc4db268SThomas Gleixner * test was either successful or failed terminally. 407cc4db268SThomas Gleixner */ 408cc4db268SThomas Gleixner if (!atomic_read(&test_runs)) 409cc4db268SThomas Gleixner return; 410cc4db268SThomas Gleixner 411cc4db268SThomas Gleixner /* 412cc4db268SThomas Gleixner * If the warp value of this CPU is 0, then the other CPU 413cc4db268SThomas Gleixner * observed time going backwards so this TSC was ahead and 414cc4db268SThomas Gleixner * needs to move backwards. 415cc4db268SThomas Gleixner */ 416cc4db268SThomas Gleixner if (!cur_max_warp) 417cc4db268SThomas Gleixner cur_max_warp = -gbl_max_warp; 418cc4db268SThomas Gleixner 419cc4db268SThomas Gleixner /* 420cc4db268SThomas Gleixner * Add the result to the previous adjustment value. 421cc4db268SThomas Gleixner * 422cc4db268SThomas Gleixner * The adjustement value is slightly off by the overhead of the 423cc4db268SThomas Gleixner * sync mechanism (observed values are ~200 TSC cycles), but this 424cc4db268SThomas Gleixner * really depends on CPU, node distance and frequency. So 425cc4db268SThomas Gleixner * compensating for this is hard to get right. Experiments show 426cc4db268SThomas Gleixner * that the warp is not longer detectable when the observed warp 427cc4db268SThomas Gleixner * value is used. In the worst case the adjustment needs to go 428cc4db268SThomas Gleixner * through a 3rd run for fine tuning. 429cc4db268SThomas Gleixner */ 430cc4db268SThomas Gleixner cur->adjusted += cur_max_warp; 431cc4db268SThomas Gleixner 432cc4db268SThomas Gleixner pr_warn("TSC ADJUST compensate: CPU%u observed %lld warp. Adjust: %lld\n", 433cc4db268SThomas Gleixner cpu, cur_max_warp, cur->adjusted); 434cc4db268SThomas Gleixner 435cc4db268SThomas Gleixner wrmsrl(MSR_IA32_TSC_ADJUST, cur->adjusted); 436cc4db268SThomas Gleixner goto retry; 437cc4db268SThomas Gleixner 438250c2277SThomas Gleixner } 4398b223bc7SThomas Gleixner 4408b223bc7SThomas Gleixner #endif /* CONFIG_SMP */ 441