1 /* 2 * check TSC synchronization. 3 * 4 * Copyright (C) 2006, Red Hat, Inc., Ingo Molnar 5 * 6 * We check whether all boot CPUs have their TSC's synchronized, 7 * print a warning if not and turn off the TSC clock-source. 8 * 9 * The warp-check is point-to-point between two CPUs, the CPU 10 * initiating the bootup is the 'source CPU', the freshly booting 11 * CPU is the 'target CPU'. 12 * 13 * Only two CPUs may participate - they can enter in any order. 14 * ( The serial nature of the boot logic and the CPU hotplug lock 15 * protects against more than 2 CPUs entering this code. ) 16 */ 17 #include <linux/topology.h> 18 #include <linux/spinlock.h> 19 #include <linux/kernel.h> 20 #include <linux/smp.h> 21 #include <linux/nmi.h> 22 #include <asm/tsc.h> 23 24 struct tsc_adjust { 25 s64 bootval; 26 s64 adjusted; 27 unsigned long nextcheck; 28 bool warned; 29 }; 30 31 static DEFINE_PER_CPU(struct tsc_adjust, tsc_adjust); 32 33 void tsc_verify_tsc_adjust(bool resume) 34 { 35 struct tsc_adjust *adj = this_cpu_ptr(&tsc_adjust); 36 s64 curval; 37 38 if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) 39 return; 40 41 /* Rate limit the MSR check */ 42 if (!resume && time_before(jiffies, adj->nextcheck)) 43 return; 44 45 adj->nextcheck = jiffies + HZ; 46 47 rdmsrl(MSR_IA32_TSC_ADJUST, curval); 48 if (adj->adjusted == curval) 49 return; 50 51 /* Restore the original value */ 52 wrmsrl(MSR_IA32_TSC_ADJUST, adj->adjusted); 53 54 if (!adj->warned || resume) { 55 pr_warn(FW_BUG "TSC ADJUST differs: CPU%u %lld --> %lld. Restoring\n", 56 smp_processor_id(), adj->adjusted, curval); 57 adj->warned = true; 58 } 59 } 60 61 static void tsc_sanitize_first_cpu(struct tsc_adjust *cur, s64 bootval, 62 unsigned int cpu, bool bootcpu) 63 { 64 /* 65 * First online CPU in a package stores the boot value in the 66 * adjustment value. This value might change later via the sync 67 * mechanism. If that fails we still can yell about boot values not 68 * being consistent. 69 * 70 * On the boot cpu we just force set the ADJUST value to 0 if it's 71 * non zero. We don't do that on non boot cpus because physical 72 * hotplug should have set the ADJUST register to a value > 0 so 73 * the TSC is in sync with the already running cpus. 74 * 75 * But we always force positive ADJUST values. Otherwise the TSC 76 * deadline timer creates an interrupt storm. We also have to 77 * prevent values > 0x7FFFFFFF as those wreckage the timer as well. 78 */ 79 if ((bootcpu && bootval != 0) || (!bootcpu && bootval < 0) || 80 (bootval > 0x7FFFFFFF)) { 81 pr_warn(FW_BUG "TSC ADJUST: CPU%u: %lld force to 0\n", cpu, 82 bootval); 83 wrmsrl(MSR_IA32_TSC_ADJUST, 0); 84 bootval = 0; 85 } 86 cur->adjusted = bootval; 87 } 88 89 #ifndef CONFIG_SMP 90 bool __init tsc_store_and_check_tsc_adjust(bool bootcpu) 91 { 92 struct tsc_adjust *cur = this_cpu_ptr(&tsc_adjust); 93 s64 bootval; 94 95 if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) 96 return false; 97 98 rdmsrl(MSR_IA32_TSC_ADJUST, bootval); 99 cur->bootval = bootval; 100 cur->nextcheck = jiffies + HZ; 101 tsc_sanitize_first_cpu(cur, bootval, smp_processor_id(), bootcpu); 102 return false; 103 } 104 105 #else /* !CONFIG_SMP */ 106 107 /* 108 * Store and check the TSC ADJUST MSR if available 109 */ 110 bool tsc_store_and_check_tsc_adjust(bool bootcpu) 111 { 112 struct tsc_adjust *ref, *cur = this_cpu_ptr(&tsc_adjust); 113 unsigned int refcpu, cpu = smp_processor_id(); 114 struct cpumask *mask; 115 s64 bootval; 116 117 if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) 118 return false; 119 120 rdmsrl(MSR_IA32_TSC_ADJUST, bootval); 121 cur->bootval = bootval; 122 cur->nextcheck = jiffies + HZ; 123 cur->warned = false; 124 125 /* 126 * Check whether this CPU is the first in a package to come up. In 127 * this case do not check the boot value against another package 128 * because the new package might have been physically hotplugged, 129 * where TSC_ADJUST is expected to be different. When called on the 130 * boot CPU topology_core_cpumask() might not be available yet. 131 */ 132 mask = topology_core_cpumask(cpu); 133 refcpu = mask ? cpumask_any_but(mask, cpu) : nr_cpu_ids; 134 135 if (refcpu >= nr_cpu_ids) { 136 tsc_sanitize_first_cpu(cur, bootval, smp_processor_id(), 137 bootcpu); 138 return false; 139 } 140 141 ref = per_cpu_ptr(&tsc_adjust, refcpu); 142 /* 143 * Compare the boot value and complain if it differs in the 144 * package. 145 */ 146 if (bootval != ref->bootval) { 147 pr_warn(FW_BUG "TSC ADJUST differs: Reference CPU%u: %lld CPU%u: %lld\n", 148 refcpu, ref->bootval, cpu, bootval); 149 } 150 /* 151 * The TSC_ADJUST values in a package must be the same. If the boot 152 * value on this newly upcoming CPU differs from the adjustment 153 * value of the already online CPU in this package, set it to that 154 * adjusted value. 155 */ 156 if (bootval != ref->adjusted) { 157 pr_warn("TSC ADJUST synchronize: Reference CPU%u: %lld CPU%u: %lld\n", 158 refcpu, ref->adjusted, cpu, bootval); 159 cur->adjusted = ref->adjusted; 160 wrmsrl(MSR_IA32_TSC_ADJUST, ref->adjusted); 161 } 162 /* 163 * We have the TSCs forced to be in sync on this package. Skip sync 164 * test: 165 */ 166 return true; 167 } 168 169 /* 170 * Entry/exit counters that make sure that both CPUs 171 * run the measurement code at once: 172 */ 173 static atomic_t start_count; 174 static atomic_t stop_count; 175 static atomic_t skip_test; 176 static atomic_t test_runs; 177 178 /* 179 * We use a raw spinlock in this exceptional case, because 180 * we want to have the fastest, inlined, non-debug version 181 * of a critical section, to be able to prove TSC time-warps: 182 */ 183 static arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED; 184 185 static cycles_t last_tsc; 186 static cycles_t max_warp; 187 static int nr_warps; 188 static int random_warps; 189 190 /* 191 * TSC-warp measurement loop running on both CPUs. This is not called 192 * if there is no TSC. 193 */ 194 static cycles_t check_tsc_warp(unsigned int timeout) 195 { 196 cycles_t start, now, prev, end, cur_max_warp = 0; 197 int i, cur_warps = 0; 198 199 start = rdtsc_ordered(); 200 /* 201 * The measurement runs for 'timeout' msecs: 202 */ 203 end = start + (cycles_t) tsc_khz * timeout; 204 now = start; 205 206 for (i = 0; ; i++) { 207 /* 208 * We take the global lock, measure TSC, save the 209 * previous TSC that was measured (possibly on 210 * another CPU) and update the previous TSC timestamp. 211 */ 212 arch_spin_lock(&sync_lock); 213 prev = last_tsc; 214 now = rdtsc_ordered(); 215 last_tsc = now; 216 arch_spin_unlock(&sync_lock); 217 218 /* 219 * Be nice every now and then (and also check whether 220 * measurement is done [we also insert a 10 million 221 * loops safety exit, so we dont lock up in case the 222 * TSC readout is totally broken]): 223 */ 224 if (unlikely(!(i & 7))) { 225 if (now > end || i > 10000000) 226 break; 227 cpu_relax(); 228 touch_nmi_watchdog(); 229 } 230 /* 231 * Outside the critical section we can now see whether 232 * we saw a time-warp of the TSC going backwards: 233 */ 234 if (unlikely(prev > now)) { 235 arch_spin_lock(&sync_lock); 236 max_warp = max(max_warp, prev - now); 237 cur_max_warp = max_warp; 238 /* 239 * Check whether this bounces back and forth. Only 240 * one CPU should observe time going backwards. 241 */ 242 if (cur_warps != nr_warps) 243 random_warps++; 244 nr_warps++; 245 cur_warps = nr_warps; 246 arch_spin_unlock(&sync_lock); 247 } 248 } 249 WARN(!(now-start), 250 "Warning: zero tsc calibration delta: %Ld [max: %Ld]\n", 251 now-start, end-start); 252 return cur_max_warp; 253 } 254 255 /* 256 * If the target CPU coming online doesn't have any of its core-siblings 257 * online, a timeout of 20msec will be used for the TSC-warp measurement 258 * loop. Otherwise a smaller timeout of 2msec will be used, as we have some 259 * information about this socket already (and this information grows as we 260 * have more and more logical-siblings in that socket). 261 * 262 * Ideally we should be able to skip the TSC sync check on the other 263 * core-siblings, if the first logical CPU in a socket passed the sync test. 264 * But as the TSC is per-logical CPU and can potentially be modified wrongly 265 * by the bios, TSC sync test for smaller duration should be able 266 * to catch such errors. Also this will catch the condition where all the 267 * cores in the socket doesn't get reset at the same time. 268 */ 269 static inline unsigned int loop_timeout(int cpu) 270 { 271 return (cpumask_weight(topology_core_cpumask(cpu)) > 1) ? 2 : 20; 272 } 273 274 /* 275 * Source CPU calls into this - it waits for the freshly booted 276 * target CPU to arrive and then starts the measurement: 277 */ 278 void check_tsc_sync_source(int cpu) 279 { 280 int cpus = 2; 281 282 /* 283 * No need to check if we already know that the TSC is not 284 * synchronized or if we have no TSC. 285 */ 286 if (unsynchronized_tsc()) 287 return; 288 289 if (tsc_clocksource_reliable) { 290 if (cpu == (nr_cpu_ids-1) || system_state != SYSTEM_BOOTING) 291 pr_info( 292 "Skipped synchronization checks as TSC is reliable.\n"); 293 return; 294 } 295 296 /* 297 * Set the maximum number of test runs to 298 * 1 if the CPU does not provide the TSC_ADJUST MSR 299 * 3 if the MSR is available, so the target can try to adjust 300 */ 301 if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) 302 atomic_set(&test_runs, 1); 303 else 304 atomic_set(&test_runs, 3); 305 retry: 306 /* 307 * Wait for the target to start or to skip the test: 308 */ 309 while (atomic_read(&start_count) != cpus - 1) { 310 if (atomic_read(&skip_test) > 0) { 311 atomic_set(&skip_test, 0); 312 return; 313 } 314 cpu_relax(); 315 } 316 317 /* 318 * Trigger the target to continue into the measurement too: 319 */ 320 atomic_inc(&start_count); 321 322 check_tsc_warp(loop_timeout(cpu)); 323 324 while (atomic_read(&stop_count) != cpus-1) 325 cpu_relax(); 326 327 /* 328 * If the test was successful set the number of runs to zero and 329 * stop. If not, decrement the number of runs an check if we can 330 * retry. In case of random warps no retry is attempted. 331 */ 332 if (!nr_warps) { 333 atomic_set(&test_runs, 0); 334 335 pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n", 336 smp_processor_id(), cpu); 337 338 } else if (atomic_dec_and_test(&test_runs) || random_warps) { 339 /* Force it to 0 if random warps brought us here */ 340 atomic_set(&test_runs, 0); 341 342 pr_warning("TSC synchronization [CPU#%d -> CPU#%d]:\n", 343 smp_processor_id(), cpu); 344 pr_warning("Measured %Ld cycles TSC warp between CPUs, " 345 "turning off TSC clock.\n", max_warp); 346 if (random_warps) 347 pr_warning("TSC warped randomly between CPUs\n"); 348 mark_tsc_unstable("check_tsc_sync_source failed"); 349 } 350 351 /* 352 * Reset it - just in case we boot another CPU later: 353 */ 354 atomic_set(&start_count, 0); 355 random_warps = 0; 356 nr_warps = 0; 357 max_warp = 0; 358 last_tsc = 0; 359 360 /* 361 * Let the target continue with the bootup: 362 */ 363 atomic_inc(&stop_count); 364 365 /* 366 * Retry, if there is a chance to do so. 367 */ 368 if (atomic_read(&test_runs) > 0) 369 goto retry; 370 } 371 372 /* 373 * Freshly booted CPUs call into this: 374 */ 375 void check_tsc_sync_target(void) 376 { 377 struct tsc_adjust *cur = this_cpu_ptr(&tsc_adjust); 378 unsigned int cpu = smp_processor_id(); 379 cycles_t cur_max_warp, gbl_max_warp; 380 int cpus = 2; 381 382 /* Also aborts if there is no TSC. */ 383 if (unsynchronized_tsc() || tsc_clocksource_reliable) 384 return; 385 386 /* 387 * Store, verify and sanitize the TSC adjust register. If 388 * successful skip the test. 389 */ 390 if (tsc_store_and_check_tsc_adjust(false)) { 391 atomic_inc(&skip_test); 392 return; 393 } 394 395 retry: 396 /* 397 * Register this CPU's participation and wait for the 398 * source CPU to start the measurement: 399 */ 400 atomic_inc(&start_count); 401 while (atomic_read(&start_count) != cpus) 402 cpu_relax(); 403 404 cur_max_warp = check_tsc_warp(loop_timeout(cpu)); 405 406 /* 407 * Store the maximum observed warp value for a potential retry: 408 */ 409 gbl_max_warp = max_warp; 410 411 /* 412 * Ok, we are done: 413 */ 414 atomic_inc(&stop_count); 415 416 /* 417 * Wait for the source CPU to print stuff: 418 */ 419 while (atomic_read(&stop_count) != cpus) 420 cpu_relax(); 421 422 /* 423 * Reset it for the next sync test: 424 */ 425 atomic_set(&stop_count, 0); 426 427 /* 428 * Check the number of remaining test runs. If not zero, the test 429 * failed and a retry with adjusted TSC is possible. If zero the 430 * test was either successful or failed terminally. 431 */ 432 if (!atomic_read(&test_runs)) 433 return; 434 435 /* 436 * If the warp value of this CPU is 0, then the other CPU 437 * observed time going backwards so this TSC was ahead and 438 * needs to move backwards. 439 */ 440 if (!cur_max_warp) 441 cur_max_warp = -gbl_max_warp; 442 443 /* 444 * Add the result to the previous adjustment value. 445 * 446 * The adjustement value is slightly off by the overhead of the 447 * sync mechanism (observed values are ~200 TSC cycles), but this 448 * really depends on CPU, node distance and frequency. So 449 * compensating for this is hard to get right. Experiments show 450 * that the warp is not longer detectable when the observed warp 451 * value is used. In the worst case the adjustment needs to go 452 * through a 3rd run for fine tuning. 453 */ 454 cur->adjusted += cur_max_warp; 455 456 /* 457 * TSC deadline timer stops working or creates an interrupt storm 458 * with adjust values < 0 and > x07ffffff. 459 * 460 * To allow adjust values > 0x7FFFFFFF we need to disable the 461 * deadline timer and use the local APIC timer, but that requires 462 * more intrusive changes and we do not have any useful information 463 * from Intel about the underlying HW wreckage yet. 464 */ 465 if (cur->adjusted < 0) 466 cur->adjusted = 0; 467 if (cur->adjusted > 0x7FFFFFFF) 468 cur->adjusted = 0x7FFFFFFF; 469 470 pr_warn("TSC ADJUST compensate: CPU%u observed %lld warp. Adjust: %lld\n", 471 cpu, cur_max_warp, cur->adjusted); 472 473 wrmsrl(MSR_IA32_TSC_ADJUST, cur->adjusted); 474 goto retry; 475 476 } 477 478 #endif /* CONFIG_SMP */ 479