1 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 2 3 #include <linux/kernel.h> 4 #include <linux/sched.h> 5 #include <linux/sched/clock.h> 6 #include <linux/init.h> 7 #include <linux/export.h> 8 #include <linux/timer.h> 9 #include <linux/acpi_pmtmr.h> 10 #include <linux/cpufreq.h> 11 #include <linux/delay.h> 12 #include <linux/clocksource.h> 13 #include <linux/percpu.h> 14 #include <linux/timex.h> 15 #include <linux/static_key.h> 16 17 #include <asm/hpet.h> 18 #include <asm/timer.h> 19 #include <asm/vgtod.h> 20 #include <asm/time.h> 21 #include <asm/delay.h> 22 #include <asm/hypervisor.h> 23 #include <asm/nmi.h> 24 #include <asm/x86_init.h> 25 #include <asm/geode.h> 26 #include <asm/apic.h> 27 #include <asm/intel-family.h> 28 29 unsigned int __read_mostly cpu_khz; /* TSC clocks / usec, not used here */ 30 EXPORT_SYMBOL(cpu_khz); 31 32 unsigned int __read_mostly tsc_khz; 33 EXPORT_SYMBOL(tsc_khz); 34 35 /* 36 * TSC can be unstable due to cpufreq or due to unsynced TSCs 37 */ 38 static int __read_mostly tsc_unstable; 39 40 /* native_sched_clock() is called before tsc_init(), so 41 we must start with the TSC soft disabled to prevent 42 erroneous rdtsc usage on !boot_cpu_has(X86_FEATURE_TSC) processors */ 43 static int __read_mostly tsc_disabled = -1; 44 45 static DEFINE_STATIC_KEY_FALSE(__use_tsc); 46 47 int tsc_clocksource_reliable; 48 49 static u32 art_to_tsc_numerator; 50 static u32 art_to_tsc_denominator; 51 static u64 art_to_tsc_offset; 52 struct clocksource *art_related_clocksource; 53 54 /* 55 * Use a ring-buffer like data structure, where a writer advances the head by 56 * writing a new data entry and a reader advances the tail when it observes a 57 * new entry. 58 * 59 * Writers are made to wait on readers until there's space to write a new 60 * entry. 61 * 62 * This means that we can always use an {offset, mul} pair to compute a ns 63 * value that is 'roughly' in the right direction, even if we're writing a new 64 * {offset, mul} pair during the clock read. 65 * 66 * The down-side is that we can no longer guarantee strict monotonicity anymore 67 * (assuming the TSC was that to begin with), because while we compute the 68 * intersection point of the two clock slopes and make sure the time is 69 * continuous at the point of switching; we can no longer guarantee a reader is 70 * strictly before or after the switch point. 71 * 72 * It does mean a reader no longer needs to disable IRQs in order to avoid 73 * CPU-Freq updates messing with his times, and similarly an NMI reader will 74 * no longer run the risk of hitting half-written state. 75 */ 76 77 struct cyc2ns { 78 struct cyc2ns_data data[2]; /* 0 + 2*24 = 48 */ 79 struct cyc2ns_data *head; /* 48 + 8 = 56 */ 80 struct cyc2ns_data *tail; /* 56 + 8 = 64 */ 81 }; /* exactly fits one cacheline */ 82 83 static DEFINE_PER_CPU_ALIGNED(struct cyc2ns, cyc2ns); 84 85 struct cyc2ns_data *cyc2ns_read_begin(void) 86 { 87 struct cyc2ns_data *head; 88 89 preempt_disable(); 90 91 head = this_cpu_read(cyc2ns.head); 92 /* 93 * Ensure we observe the entry when we observe the pointer to it. 94 * matches the wmb from cyc2ns_write_end(). 95 */ 96 smp_read_barrier_depends(); 97 head->__count++; 98 barrier(); 99 100 return head; 101 } 102 103 void cyc2ns_read_end(struct cyc2ns_data *head) 104 { 105 barrier(); 106 /* 107 * If we're the outer most nested read; update the tail pointer 108 * when we're done. This notifies possible pending writers 109 * that we've observed the head pointer and that the other 110 * entry is now free. 111 */ 112 if (!--head->__count) { 113 /* 114 * x86-TSO does not reorder writes with older reads; 115 * therefore once this write becomes visible to another 116 * cpu, we must be finished reading the cyc2ns_data. 117 * 118 * matches with cyc2ns_write_begin(). 119 */ 120 this_cpu_write(cyc2ns.tail, head); 121 } 122 preempt_enable(); 123 } 124 125 /* 126 * Begin writing a new @data entry for @cpu. 127 * 128 * Assumes some sort of write side lock; currently 'provided' by the assumption 129 * that cpufreq will call its notifiers sequentially. 130 */ 131 static struct cyc2ns_data *cyc2ns_write_begin(int cpu) 132 { 133 struct cyc2ns *c2n = &per_cpu(cyc2ns, cpu); 134 struct cyc2ns_data *data = c2n->data; 135 136 if (data == c2n->head) 137 data++; 138 139 /* XXX send an IPI to @cpu in order to guarantee a read? */ 140 141 /* 142 * When we observe the tail write from cyc2ns_read_end(), 143 * the cpu must be done with that entry and its safe 144 * to start writing to it. 145 */ 146 while (c2n->tail == data) 147 cpu_relax(); 148 149 return data; 150 } 151 152 static void cyc2ns_write_end(int cpu, struct cyc2ns_data *data) 153 { 154 struct cyc2ns *c2n = &per_cpu(cyc2ns, cpu); 155 156 /* 157 * Ensure the @data writes are visible before we publish the 158 * entry. Matches the data-depencency in cyc2ns_read_begin(). 159 */ 160 smp_wmb(); 161 162 ACCESS_ONCE(c2n->head) = data; 163 } 164 165 /* 166 * Accelerators for sched_clock() 167 * convert from cycles(64bits) => nanoseconds (64bits) 168 * basic equation: 169 * ns = cycles / (freq / ns_per_sec) 170 * ns = cycles * (ns_per_sec / freq) 171 * ns = cycles * (10^9 / (cpu_khz * 10^3)) 172 * ns = cycles * (10^6 / cpu_khz) 173 * 174 * Then we use scaling math (suggested by george@mvista.com) to get: 175 * ns = cycles * (10^6 * SC / cpu_khz) / SC 176 * ns = cycles * cyc2ns_scale / SC 177 * 178 * And since SC is a constant power of two, we can convert the div 179 * into a shift. The larger SC is, the more accurate the conversion, but 180 * cyc2ns_scale needs to be a 32-bit value so that 32-bit multiplication 181 * (64-bit result) can be used. 182 * 183 * We can use khz divisor instead of mhz to keep a better precision. 184 * (mathieu.desnoyers@polymtl.ca) 185 * 186 * -johnstul@us.ibm.com "math is hard, lets go shopping!" 187 */ 188 189 static void cyc2ns_data_init(struct cyc2ns_data *data) 190 { 191 data->cyc2ns_mul = 0; 192 data->cyc2ns_shift = 0; 193 data->cyc2ns_offset = 0; 194 data->__count = 0; 195 } 196 197 static void cyc2ns_init(int cpu) 198 { 199 struct cyc2ns *c2n = &per_cpu(cyc2ns, cpu); 200 201 cyc2ns_data_init(&c2n->data[0]); 202 cyc2ns_data_init(&c2n->data[1]); 203 204 c2n->head = c2n->data; 205 c2n->tail = c2n->data; 206 } 207 208 static inline unsigned long long cycles_2_ns(unsigned long long cyc) 209 { 210 struct cyc2ns_data *data, *tail; 211 unsigned long long ns; 212 213 /* 214 * See cyc2ns_read_*() for details; replicated in order to avoid 215 * an extra few instructions that came with the abstraction. 216 * Notable, it allows us to only do the __count and tail update 217 * dance when its actually needed. 218 */ 219 220 preempt_disable_notrace(); 221 data = this_cpu_read(cyc2ns.head); 222 tail = this_cpu_read(cyc2ns.tail); 223 224 if (likely(data == tail)) { 225 ns = data->cyc2ns_offset; 226 ns += mul_u64_u32_shr(cyc, data->cyc2ns_mul, data->cyc2ns_shift); 227 } else { 228 data->__count++; 229 230 barrier(); 231 232 ns = data->cyc2ns_offset; 233 ns += mul_u64_u32_shr(cyc, data->cyc2ns_mul, data->cyc2ns_shift); 234 235 barrier(); 236 237 if (!--data->__count) 238 this_cpu_write(cyc2ns.tail, data); 239 } 240 preempt_enable_notrace(); 241 242 return ns; 243 } 244 245 static void set_cyc2ns_scale(unsigned long khz, int cpu) 246 { 247 unsigned long long tsc_now, ns_now; 248 struct cyc2ns_data *data; 249 unsigned long flags; 250 251 local_irq_save(flags); 252 sched_clock_idle_sleep_event(); 253 254 if (!khz) 255 goto done; 256 257 data = cyc2ns_write_begin(cpu); 258 259 tsc_now = rdtsc(); 260 ns_now = cycles_2_ns(tsc_now); 261 262 /* 263 * Compute a new multiplier as per the above comment and ensure our 264 * time function is continuous; see the comment near struct 265 * cyc2ns_data. 266 */ 267 clocks_calc_mult_shift(&data->cyc2ns_mul, &data->cyc2ns_shift, khz, 268 NSEC_PER_MSEC, 0); 269 270 /* 271 * cyc2ns_shift is exported via arch_perf_update_userpage() where it is 272 * not expected to be greater than 31 due to the original published 273 * conversion algorithm shifting a 32-bit value (now specifies a 64-bit 274 * value) - refer perf_event_mmap_page documentation in perf_event.h. 275 */ 276 if (data->cyc2ns_shift == 32) { 277 data->cyc2ns_shift = 31; 278 data->cyc2ns_mul >>= 1; 279 } 280 281 data->cyc2ns_offset = ns_now - 282 mul_u64_u32_shr(tsc_now, data->cyc2ns_mul, data->cyc2ns_shift); 283 284 cyc2ns_write_end(cpu, data); 285 286 done: 287 sched_clock_idle_wakeup_event(0); 288 local_irq_restore(flags); 289 } 290 /* 291 * Scheduler clock - returns current time in nanosec units. 292 */ 293 u64 native_sched_clock(void) 294 { 295 if (static_branch_likely(&__use_tsc)) { 296 u64 tsc_now = rdtsc(); 297 298 /* return the value in ns */ 299 return cycles_2_ns(tsc_now); 300 } 301 302 /* 303 * Fall back to jiffies if there's no TSC available: 304 * ( But note that we still use it if the TSC is marked 305 * unstable. We do this because unlike Time Of Day, 306 * the scheduler clock tolerates small errors and it's 307 * very important for it to be as fast as the platform 308 * can achieve it. ) 309 */ 310 311 /* No locking but a rare wrong value is not a big deal: */ 312 return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ); 313 } 314 315 /* 316 * Generate a sched_clock if you already have a TSC value. 317 */ 318 u64 native_sched_clock_from_tsc(u64 tsc) 319 { 320 return cycles_2_ns(tsc); 321 } 322 323 /* We need to define a real function for sched_clock, to override the 324 weak default version */ 325 #ifdef CONFIG_PARAVIRT 326 unsigned long long sched_clock(void) 327 { 328 return paravirt_sched_clock(); 329 } 330 331 bool using_native_sched_clock(void) 332 { 333 return pv_time_ops.sched_clock == native_sched_clock; 334 } 335 #else 336 unsigned long long 337 sched_clock(void) __attribute__((alias("native_sched_clock"))); 338 339 bool using_native_sched_clock(void) { return true; } 340 #endif 341 342 int check_tsc_unstable(void) 343 { 344 return tsc_unstable; 345 } 346 EXPORT_SYMBOL_GPL(check_tsc_unstable); 347 348 #ifdef CONFIG_X86_TSC 349 int __init notsc_setup(char *str) 350 { 351 pr_warn("Kernel compiled with CONFIG_X86_TSC, cannot disable TSC completely\n"); 352 tsc_disabled = 1; 353 return 1; 354 } 355 #else 356 /* 357 * disable flag for tsc. Takes effect by clearing the TSC cpu flag 358 * in cpu/common.c 359 */ 360 int __init notsc_setup(char *str) 361 { 362 setup_clear_cpu_cap(X86_FEATURE_TSC); 363 return 1; 364 } 365 #endif 366 367 __setup("notsc", notsc_setup); 368 369 static int no_sched_irq_time; 370 371 static int __init tsc_setup(char *str) 372 { 373 if (!strcmp(str, "reliable")) 374 tsc_clocksource_reliable = 1; 375 if (!strncmp(str, "noirqtime", 9)) 376 no_sched_irq_time = 1; 377 return 1; 378 } 379 380 __setup("tsc=", tsc_setup); 381 382 #define MAX_RETRIES 5 383 #define SMI_TRESHOLD 50000 384 385 /* 386 * Read TSC and the reference counters. Take care of SMI disturbance 387 */ 388 static u64 tsc_read_refs(u64 *p, int hpet) 389 { 390 u64 t1, t2; 391 int i; 392 393 for (i = 0; i < MAX_RETRIES; i++) { 394 t1 = get_cycles(); 395 if (hpet) 396 *p = hpet_readl(HPET_COUNTER) & 0xFFFFFFFF; 397 else 398 *p = acpi_pm_read_early(); 399 t2 = get_cycles(); 400 if ((t2 - t1) < SMI_TRESHOLD) 401 return t2; 402 } 403 return ULLONG_MAX; 404 } 405 406 /* 407 * Calculate the TSC frequency from HPET reference 408 */ 409 static unsigned long calc_hpet_ref(u64 deltatsc, u64 hpet1, u64 hpet2) 410 { 411 u64 tmp; 412 413 if (hpet2 < hpet1) 414 hpet2 += 0x100000000ULL; 415 hpet2 -= hpet1; 416 tmp = ((u64)hpet2 * hpet_readl(HPET_PERIOD)); 417 do_div(tmp, 1000000); 418 do_div(deltatsc, tmp); 419 420 return (unsigned long) deltatsc; 421 } 422 423 /* 424 * Calculate the TSC frequency from PMTimer reference 425 */ 426 static unsigned long calc_pmtimer_ref(u64 deltatsc, u64 pm1, u64 pm2) 427 { 428 u64 tmp; 429 430 if (!pm1 && !pm2) 431 return ULONG_MAX; 432 433 if (pm2 < pm1) 434 pm2 += (u64)ACPI_PM_OVRRUN; 435 pm2 -= pm1; 436 tmp = pm2 * 1000000000LL; 437 do_div(tmp, PMTMR_TICKS_PER_SEC); 438 do_div(deltatsc, tmp); 439 440 return (unsigned long) deltatsc; 441 } 442 443 #define CAL_MS 10 444 #define CAL_LATCH (PIT_TICK_RATE / (1000 / CAL_MS)) 445 #define CAL_PIT_LOOPS 1000 446 447 #define CAL2_MS 50 448 #define CAL2_LATCH (PIT_TICK_RATE / (1000 / CAL2_MS)) 449 #define CAL2_PIT_LOOPS 5000 450 451 452 /* 453 * Try to calibrate the TSC against the Programmable 454 * Interrupt Timer and return the frequency of the TSC 455 * in kHz. 456 * 457 * Return ULONG_MAX on failure to calibrate. 458 */ 459 static unsigned long pit_calibrate_tsc(u32 latch, unsigned long ms, int loopmin) 460 { 461 u64 tsc, t1, t2, delta; 462 unsigned long tscmin, tscmax; 463 int pitcnt; 464 465 /* Set the Gate high, disable speaker */ 466 outb((inb(0x61) & ~0x02) | 0x01, 0x61); 467 468 /* 469 * Setup CTC channel 2* for mode 0, (interrupt on terminal 470 * count mode), binary count. Set the latch register to 50ms 471 * (LSB then MSB) to begin countdown. 472 */ 473 outb(0xb0, 0x43); 474 outb(latch & 0xff, 0x42); 475 outb(latch >> 8, 0x42); 476 477 tsc = t1 = t2 = get_cycles(); 478 479 pitcnt = 0; 480 tscmax = 0; 481 tscmin = ULONG_MAX; 482 while ((inb(0x61) & 0x20) == 0) { 483 t2 = get_cycles(); 484 delta = t2 - tsc; 485 tsc = t2; 486 if ((unsigned long) delta < tscmin) 487 tscmin = (unsigned int) delta; 488 if ((unsigned long) delta > tscmax) 489 tscmax = (unsigned int) delta; 490 pitcnt++; 491 } 492 493 /* 494 * Sanity checks: 495 * 496 * If we were not able to read the PIT more than loopmin 497 * times, then we have been hit by a massive SMI 498 * 499 * If the maximum is 10 times larger than the minimum, 500 * then we got hit by an SMI as well. 501 */ 502 if (pitcnt < loopmin || tscmax > 10 * tscmin) 503 return ULONG_MAX; 504 505 /* Calculate the PIT value */ 506 delta = t2 - t1; 507 do_div(delta, ms); 508 return delta; 509 } 510 511 /* 512 * This reads the current MSB of the PIT counter, and 513 * checks if we are running on sufficiently fast and 514 * non-virtualized hardware. 515 * 516 * Our expectations are: 517 * 518 * - the PIT is running at roughly 1.19MHz 519 * 520 * - each IO is going to take about 1us on real hardware, 521 * but we allow it to be much faster (by a factor of 10) or 522 * _slightly_ slower (ie we allow up to a 2us read+counter 523 * update - anything else implies a unacceptably slow CPU 524 * or PIT for the fast calibration to work. 525 * 526 * - with 256 PIT ticks to read the value, we have 214us to 527 * see the same MSB (and overhead like doing a single TSC 528 * read per MSB value etc). 529 * 530 * - We're doing 2 reads per loop (LSB, MSB), and we expect 531 * them each to take about a microsecond on real hardware. 532 * So we expect a count value of around 100. But we'll be 533 * generous, and accept anything over 50. 534 * 535 * - if the PIT is stuck, and we see *many* more reads, we 536 * return early (and the next caller of pit_expect_msb() 537 * then consider it a failure when they don't see the 538 * next expected value). 539 * 540 * These expectations mean that we know that we have seen the 541 * transition from one expected value to another with a fairly 542 * high accuracy, and we didn't miss any events. We can thus 543 * use the TSC value at the transitions to calculate a pretty 544 * good value for the TSC frequencty. 545 */ 546 static inline int pit_verify_msb(unsigned char val) 547 { 548 /* Ignore LSB */ 549 inb(0x42); 550 return inb(0x42) == val; 551 } 552 553 static inline int pit_expect_msb(unsigned char val, u64 *tscp, unsigned long *deltap) 554 { 555 int count; 556 u64 tsc = 0, prev_tsc = 0; 557 558 for (count = 0; count < 50000; count++) { 559 if (!pit_verify_msb(val)) 560 break; 561 prev_tsc = tsc; 562 tsc = get_cycles(); 563 } 564 *deltap = get_cycles() - prev_tsc; 565 *tscp = tsc; 566 567 /* 568 * We require _some_ success, but the quality control 569 * will be based on the error terms on the TSC values. 570 */ 571 return count > 5; 572 } 573 574 /* 575 * How many MSB values do we want to see? We aim for 576 * a maximum error rate of 500ppm (in practice the 577 * real error is much smaller), but refuse to spend 578 * more than 50ms on it. 579 */ 580 #define MAX_QUICK_PIT_MS 50 581 #define MAX_QUICK_PIT_ITERATIONS (MAX_QUICK_PIT_MS * PIT_TICK_RATE / 1000 / 256) 582 583 static unsigned long quick_pit_calibrate(void) 584 { 585 int i; 586 u64 tsc, delta; 587 unsigned long d1, d2; 588 589 /* Set the Gate high, disable speaker */ 590 outb((inb(0x61) & ~0x02) | 0x01, 0x61); 591 592 /* 593 * Counter 2, mode 0 (one-shot), binary count 594 * 595 * NOTE! Mode 2 decrements by two (and then the 596 * output is flipped each time, giving the same 597 * final output frequency as a decrement-by-one), 598 * so mode 0 is much better when looking at the 599 * individual counts. 600 */ 601 outb(0xb0, 0x43); 602 603 /* Start at 0xffff */ 604 outb(0xff, 0x42); 605 outb(0xff, 0x42); 606 607 /* 608 * The PIT starts counting at the next edge, so we 609 * need to delay for a microsecond. The easiest way 610 * to do that is to just read back the 16-bit counter 611 * once from the PIT. 612 */ 613 pit_verify_msb(0); 614 615 if (pit_expect_msb(0xff, &tsc, &d1)) { 616 for (i = 1; i <= MAX_QUICK_PIT_ITERATIONS; i++) { 617 if (!pit_expect_msb(0xff-i, &delta, &d2)) 618 break; 619 620 delta -= tsc; 621 622 /* 623 * Extrapolate the error and fail fast if the error will 624 * never be below 500 ppm. 625 */ 626 if (i == 1 && 627 d1 + d2 >= (delta * MAX_QUICK_PIT_ITERATIONS) >> 11) 628 return 0; 629 630 /* 631 * Iterate until the error is less than 500 ppm 632 */ 633 if (d1+d2 >= delta >> 11) 634 continue; 635 636 /* 637 * Check the PIT one more time to verify that 638 * all TSC reads were stable wrt the PIT. 639 * 640 * This also guarantees serialization of the 641 * last cycle read ('d2') in pit_expect_msb. 642 */ 643 if (!pit_verify_msb(0xfe - i)) 644 break; 645 goto success; 646 } 647 } 648 pr_info("Fast TSC calibration failed\n"); 649 return 0; 650 651 success: 652 /* 653 * Ok, if we get here, then we've seen the 654 * MSB of the PIT decrement 'i' times, and the 655 * error has shrunk to less than 500 ppm. 656 * 657 * As a result, we can depend on there not being 658 * any odd delays anywhere, and the TSC reads are 659 * reliable (within the error). 660 * 661 * kHz = ticks / time-in-seconds / 1000; 662 * kHz = (t2 - t1) / (I * 256 / PIT_TICK_RATE) / 1000 663 * kHz = ((t2 - t1) * PIT_TICK_RATE) / (I * 256 * 1000) 664 */ 665 delta *= PIT_TICK_RATE; 666 do_div(delta, i*256*1000); 667 pr_info("Fast TSC calibration using PIT\n"); 668 return delta; 669 } 670 671 /** 672 * native_calibrate_tsc 673 * Determine TSC frequency via CPUID, else return 0. 674 */ 675 unsigned long native_calibrate_tsc(void) 676 { 677 unsigned int eax_denominator, ebx_numerator, ecx_hz, edx; 678 unsigned int crystal_khz; 679 680 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) 681 return 0; 682 683 if (boot_cpu_data.cpuid_level < 0x15) 684 return 0; 685 686 eax_denominator = ebx_numerator = ecx_hz = edx = 0; 687 688 /* CPUID 15H TSC/Crystal ratio, plus optionally Crystal Hz */ 689 cpuid(0x15, &eax_denominator, &ebx_numerator, &ecx_hz, &edx); 690 691 if (ebx_numerator == 0 || eax_denominator == 0) 692 return 0; 693 694 crystal_khz = ecx_hz / 1000; 695 696 if (crystal_khz == 0) { 697 switch (boot_cpu_data.x86_model) { 698 case INTEL_FAM6_SKYLAKE_MOBILE: 699 case INTEL_FAM6_SKYLAKE_DESKTOP: 700 case INTEL_FAM6_KABYLAKE_MOBILE: 701 case INTEL_FAM6_KABYLAKE_DESKTOP: 702 crystal_khz = 24000; /* 24.0 MHz */ 703 break; 704 case INTEL_FAM6_SKYLAKE_X: 705 case INTEL_FAM6_ATOM_DENVERTON: 706 crystal_khz = 25000; /* 25.0 MHz */ 707 break; 708 case INTEL_FAM6_ATOM_GOLDMONT: 709 crystal_khz = 19200; /* 19.2 MHz */ 710 break; 711 } 712 } 713 714 /* 715 * TSC frequency determined by CPUID is a "hardware reported" 716 * frequency and is the most accurate one so far we have. This 717 * is considered a known frequency. 718 */ 719 setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ); 720 721 /* 722 * For Atom SoCs TSC is the only reliable clocksource. 723 * Mark TSC reliable so no watchdog on it. 724 */ 725 if (boot_cpu_data.x86_model == INTEL_FAM6_ATOM_GOLDMONT) 726 setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE); 727 728 return crystal_khz * ebx_numerator / eax_denominator; 729 } 730 731 static unsigned long cpu_khz_from_cpuid(void) 732 { 733 unsigned int eax_base_mhz, ebx_max_mhz, ecx_bus_mhz, edx; 734 735 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) 736 return 0; 737 738 if (boot_cpu_data.cpuid_level < 0x16) 739 return 0; 740 741 eax_base_mhz = ebx_max_mhz = ecx_bus_mhz = edx = 0; 742 743 cpuid(0x16, &eax_base_mhz, &ebx_max_mhz, &ecx_bus_mhz, &edx); 744 745 return eax_base_mhz * 1000; 746 } 747 748 /** 749 * native_calibrate_cpu - calibrate the cpu on boot 750 */ 751 unsigned long native_calibrate_cpu(void) 752 { 753 u64 tsc1, tsc2, delta, ref1, ref2; 754 unsigned long tsc_pit_min = ULONG_MAX, tsc_ref_min = ULONG_MAX; 755 unsigned long flags, latch, ms, fast_calibrate; 756 int hpet = is_hpet_enabled(), i, loopmin; 757 758 fast_calibrate = cpu_khz_from_cpuid(); 759 if (fast_calibrate) 760 return fast_calibrate; 761 762 fast_calibrate = cpu_khz_from_msr(); 763 if (fast_calibrate) 764 return fast_calibrate; 765 766 local_irq_save(flags); 767 fast_calibrate = quick_pit_calibrate(); 768 local_irq_restore(flags); 769 if (fast_calibrate) 770 return fast_calibrate; 771 772 /* 773 * Run 5 calibration loops to get the lowest frequency value 774 * (the best estimate). We use two different calibration modes 775 * here: 776 * 777 * 1) PIT loop. We set the PIT Channel 2 to oneshot mode and 778 * load a timeout of 50ms. We read the time right after we 779 * started the timer and wait until the PIT count down reaches 780 * zero. In each wait loop iteration we read the TSC and check 781 * the delta to the previous read. We keep track of the min 782 * and max values of that delta. The delta is mostly defined 783 * by the IO time of the PIT access, so we can detect when a 784 * SMI/SMM disturbance happened between the two reads. If the 785 * maximum time is significantly larger than the minimum time, 786 * then we discard the result and have another try. 787 * 788 * 2) Reference counter. If available we use the HPET or the 789 * PMTIMER as a reference to check the sanity of that value. 790 * We use separate TSC readouts and check inside of the 791 * reference read for a SMI/SMM disturbance. We dicard 792 * disturbed values here as well. We do that around the PIT 793 * calibration delay loop as we have to wait for a certain 794 * amount of time anyway. 795 */ 796 797 /* Preset PIT loop values */ 798 latch = CAL_LATCH; 799 ms = CAL_MS; 800 loopmin = CAL_PIT_LOOPS; 801 802 for (i = 0; i < 3; i++) { 803 unsigned long tsc_pit_khz; 804 805 /* 806 * Read the start value and the reference count of 807 * hpet/pmtimer when available. Then do the PIT 808 * calibration, which will take at least 50ms, and 809 * read the end value. 810 */ 811 local_irq_save(flags); 812 tsc1 = tsc_read_refs(&ref1, hpet); 813 tsc_pit_khz = pit_calibrate_tsc(latch, ms, loopmin); 814 tsc2 = tsc_read_refs(&ref2, hpet); 815 local_irq_restore(flags); 816 817 /* Pick the lowest PIT TSC calibration so far */ 818 tsc_pit_min = min(tsc_pit_min, tsc_pit_khz); 819 820 /* hpet or pmtimer available ? */ 821 if (ref1 == ref2) 822 continue; 823 824 /* Check, whether the sampling was disturbed by an SMI */ 825 if (tsc1 == ULLONG_MAX || tsc2 == ULLONG_MAX) 826 continue; 827 828 tsc2 = (tsc2 - tsc1) * 1000000LL; 829 if (hpet) 830 tsc2 = calc_hpet_ref(tsc2, ref1, ref2); 831 else 832 tsc2 = calc_pmtimer_ref(tsc2, ref1, ref2); 833 834 tsc_ref_min = min(tsc_ref_min, (unsigned long) tsc2); 835 836 /* Check the reference deviation */ 837 delta = ((u64) tsc_pit_min) * 100; 838 do_div(delta, tsc_ref_min); 839 840 /* 841 * If both calibration results are inside a 10% window 842 * then we can be sure, that the calibration 843 * succeeded. We break out of the loop right away. We 844 * use the reference value, as it is more precise. 845 */ 846 if (delta >= 90 && delta <= 110) { 847 pr_info("PIT calibration matches %s. %d loops\n", 848 hpet ? "HPET" : "PMTIMER", i + 1); 849 return tsc_ref_min; 850 } 851 852 /* 853 * Check whether PIT failed more than once. This 854 * happens in virtualized environments. We need to 855 * give the virtual PC a slightly longer timeframe for 856 * the HPET/PMTIMER to make the result precise. 857 */ 858 if (i == 1 && tsc_pit_min == ULONG_MAX) { 859 latch = CAL2_LATCH; 860 ms = CAL2_MS; 861 loopmin = CAL2_PIT_LOOPS; 862 } 863 } 864 865 /* 866 * Now check the results. 867 */ 868 if (tsc_pit_min == ULONG_MAX) { 869 /* PIT gave no useful value */ 870 pr_warn("Unable to calibrate against PIT\n"); 871 872 /* We don't have an alternative source, disable TSC */ 873 if (!hpet && !ref1 && !ref2) { 874 pr_notice("No reference (HPET/PMTIMER) available\n"); 875 return 0; 876 } 877 878 /* The alternative source failed as well, disable TSC */ 879 if (tsc_ref_min == ULONG_MAX) { 880 pr_warn("HPET/PMTIMER calibration failed\n"); 881 return 0; 882 } 883 884 /* Use the alternative source */ 885 pr_info("using %s reference calibration\n", 886 hpet ? "HPET" : "PMTIMER"); 887 888 return tsc_ref_min; 889 } 890 891 /* We don't have an alternative source, use the PIT calibration value */ 892 if (!hpet && !ref1 && !ref2) { 893 pr_info("Using PIT calibration value\n"); 894 return tsc_pit_min; 895 } 896 897 /* The alternative source failed, use the PIT calibration value */ 898 if (tsc_ref_min == ULONG_MAX) { 899 pr_warn("HPET/PMTIMER calibration failed. Using PIT calibration.\n"); 900 return tsc_pit_min; 901 } 902 903 /* 904 * The calibration values differ too much. In doubt, we use 905 * the PIT value as we know that there are PMTIMERs around 906 * running at double speed. At least we let the user know: 907 */ 908 pr_warn("PIT calibration deviates from %s: %lu %lu\n", 909 hpet ? "HPET" : "PMTIMER", tsc_pit_min, tsc_ref_min); 910 pr_info("Using PIT calibration value\n"); 911 return tsc_pit_min; 912 } 913 914 int recalibrate_cpu_khz(void) 915 { 916 #ifndef CONFIG_SMP 917 unsigned long cpu_khz_old = cpu_khz; 918 919 if (!boot_cpu_has(X86_FEATURE_TSC)) 920 return -ENODEV; 921 922 cpu_khz = x86_platform.calibrate_cpu(); 923 tsc_khz = x86_platform.calibrate_tsc(); 924 if (tsc_khz == 0) 925 tsc_khz = cpu_khz; 926 else if (abs(cpu_khz - tsc_khz) * 10 > tsc_khz) 927 cpu_khz = tsc_khz; 928 cpu_data(0).loops_per_jiffy = cpufreq_scale(cpu_data(0).loops_per_jiffy, 929 cpu_khz_old, cpu_khz); 930 931 return 0; 932 #else 933 return -ENODEV; 934 #endif 935 } 936 937 EXPORT_SYMBOL(recalibrate_cpu_khz); 938 939 940 static unsigned long long cyc2ns_suspend; 941 942 void tsc_save_sched_clock_state(void) 943 { 944 if (!sched_clock_stable()) 945 return; 946 947 cyc2ns_suspend = sched_clock(); 948 } 949 950 /* 951 * Even on processors with invariant TSC, TSC gets reset in some the 952 * ACPI system sleep states. And in some systems BIOS seem to reinit TSC to 953 * arbitrary value (still sync'd across cpu's) during resume from such sleep 954 * states. To cope up with this, recompute the cyc2ns_offset for each cpu so 955 * that sched_clock() continues from the point where it was left off during 956 * suspend. 957 */ 958 void tsc_restore_sched_clock_state(void) 959 { 960 unsigned long long offset; 961 unsigned long flags; 962 int cpu; 963 964 if (!sched_clock_stable()) 965 return; 966 967 local_irq_save(flags); 968 969 /* 970 * We're coming out of suspend, there's no concurrency yet; don't 971 * bother being nice about the RCU stuff, just write to both 972 * data fields. 973 */ 974 975 this_cpu_write(cyc2ns.data[0].cyc2ns_offset, 0); 976 this_cpu_write(cyc2ns.data[1].cyc2ns_offset, 0); 977 978 offset = cyc2ns_suspend - sched_clock(); 979 980 for_each_possible_cpu(cpu) { 981 per_cpu(cyc2ns.data[0].cyc2ns_offset, cpu) = offset; 982 per_cpu(cyc2ns.data[1].cyc2ns_offset, cpu) = offset; 983 } 984 985 local_irq_restore(flags); 986 } 987 988 #ifdef CONFIG_CPU_FREQ 989 990 /* Frequency scaling support. Adjust the TSC based timer when the cpu frequency 991 * changes. 992 * 993 * RED-PEN: On SMP we assume all CPUs run with the same frequency. It's 994 * not that important because current Opteron setups do not support 995 * scaling on SMP anyroads. 996 * 997 * Should fix up last_tsc too. Currently gettimeofday in the 998 * first tick after the change will be slightly wrong. 999 */ 1000 1001 static unsigned int ref_freq; 1002 static unsigned long loops_per_jiffy_ref; 1003 static unsigned long tsc_khz_ref; 1004 1005 static int time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, 1006 void *data) 1007 { 1008 struct cpufreq_freqs *freq = data; 1009 unsigned long *lpj; 1010 1011 lpj = &boot_cpu_data.loops_per_jiffy; 1012 #ifdef CONFIG_SMP 1013 if (!(freq->flags & CPUFREQ_CONST_LOOPS)) 1014 lpj = &cpu_data(freq->cpu).loops_per_jiffy; 1015 #endif 1016 1017 if (!ref_freq) { 1018 ref_freq = freq->old; 1019 loops_per_jiffy_ref = *lpj; 1020 tsc_khz_ref = tsc_khz; 1021 } 1022 if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) || 1023 (val == CPUFREQ_POSTCHANGE && freq->old > freq->new)) { 1024 *lpj = cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new); 1025 1026 tsc_khz = cpufreq_scale(tsc_khz_ref, ref_freq, freq->new); 1027 if (!(freq->flags & CPUFREQ_CONST_LOOPS)) 1028 mark_tsc_unstable("cpufreq changes"); 1029 1030 set_cyc2ns_scale(tsc_khz, freq->cpu); 1031 } 1032 1033 return 0; 1034 } 1035 1036 static struct notifier_block time_cpufreq_notifier_block = { 1037 .notifier_call = time_cpufreq_notifier 1038 }; 1039 1040 static int __init cpufreq_register_tsc_scaling(void) 1041 { 1042 if (!boot_cpu_has(X86_FEATURE_TSC)) 1043 return 0; 1044 if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) 1045 return 0; 1046 cpufreq_register_notifier(&time_cpufreq_notifier_block, 1047 CPUFREQ_TRANSITION_NOTIFIER); 1048 return 0; 1049 } 1050 1051 core_initcall(cpufreq_register_tsc_scaling); 1052 1053 #endif /* CONFIG_CPU_FREQ */ 1054 1055 #define ART_CPUID_LEAF (0x15) 1056 #define ART_MIN_DENOMINATOR (1) 1057 1058 1059 /* 1060 * If ART is present detect the numerator:denominator to convert to TSC 1061 */ 1062 static void detect_art(void) 1063 { 1064 unsigned int unused[2]; 1065 1066 if (boot_cpu_data.cpuid_level < ART_CPUID_LEAF) 1067 return; 1068 1069 /* Don't enable ART in a VM, non-stop TSC and TSC_ADJUST required */ 1070 if (boot_cpu_has(X86_FEATURE_HYPERVISOR) || 1071 !boot_cpu_has(X86_FEATURE_NONSTOP_TSC) || 1072 !boot_cpu_has(X86_FEATURE_TSC_ADJUST)) 1073 return; 1074 1075 cpuid(ART_CPUID_LEAF, &art_to_tsc_denominator, 1076 &art_to_tsc_numerator, unused, unused+1); 1077 1078 if (art_to_tsc_denominator < ART_MIN_DENOMINATOR) 1079 return; 1080 1081 rdmsrl(MSR_IA32_TSC_ADJUST, art_to_tsc_offset); 1082 1083 /* Make this sticky over multiple CPU init calls */ 1084 setup_force_cpu_cap(X86_FEATURE_ART); 1085 } 1086 1087 1088 /* clocksource code */ 1089 1090 static struct clocksource clocksource_tsc; 1091 1092 static void tsc_resume(struct clocksource *cs) 1093 { 1094 tsc_verify_tsc_adjust(true); 1095 } 1096 1097 /* 1098 * We used to compare the TSC to the cycle_last value in the clocksource 1099 * structure to avoid a nasty time-warp. This can be observed in a 1100 * very small window right after one CPU updated cycle_last under 1101 * xtime/vsyscall_gtod lock and the other CPU reads a TSC value which 1102 * is smaller than the cycle_last reference value due to a TSC which 1103 * is slighty behind. This delta is nowhere else observable, but in 1104 * that case it results in a forward time jump in the range of hours 1105 * due to the unsigned delta calculation of the time keeping core 1106 * code, which is necessary to support wrapping clocksources like pm 1107 * timer. 1108 * 1109 * This sanity check is now done in the core timekeeping code. 1110 * checking the result of read_tsc() - cycle_last for being negative. 1111 * That works because CLOCKSOURCE_MASK(64) does not mask out any bit. 1112 */ 1113 static u64 read_tsc(struct clocksource *cs) 1114 { 1115 return (u64)rdtsc_ordered(); 1116 } 1117 1118 static void tsc_cs_mark_unstable(struct clocksource *cs) 1119 { 1120 if (tsc_unstable) 1121 return; 1122 1123 tsc_unstable = 1; 1124 if (using_native_sched_clock()) 1125 clear_sched_clock_stable(); 1126 disable_sched_clock_irqtime(); 1127 pr_info("Marking TSC unstable due to clocksource watchdog\n"); 1128 } 1129 1130 /* 1131 * .mask MUST be CLOCKSOURCE_MASK(64). See comment above read_tsc() 1132 */ 1133 static struct clocksource clocksource_tsc = { 1134 .name = "tsc", 1135 .rating = 300, 1136 .read = read_tsc, 1137 .mask = CLOCKSOURCE_MASK(64), 1138 .flags = CLOCK_SOURCE_IS_CONTINUOUS | 1139 CLOCK_SOURCE_MUST_VERIFY, 1140 .archdata = { .vclock_mode = VCLOCK_TSC }, 1141 .resume = tsc_resume, 1142 .mark_unstable = tsc_cs_mark_unstable, 1143 }; 1144 1145 void mark_tsc_unstable(char *reason) 1146 { 1147 if (tsc_unstable) 1148 return; 1149 1150 tsc_unstable = 1; 1151 if (using_native_sched_clock()) 1152 clear_sched_clock_stable(); 1153 disable_sched_clock_irqtime(); 1154 pr_info("Marking TSC unstable due to %s\n", reason); 1155 /* Change only the rating, when not registered */ 1156 if (clocksource_tsc.mult) { 1157 clocksource_mark_unstable(&clocksource_tsc); 1158 } else { 1159 clocksource_tsc.flags |= CLOCK_SOURCE_UNSTABLE; 1160 clocksource_tsc.rating = 0; 1161 } 1162 } 1163 1164 EXPORT_SYMBOL_GPL(mark_tsc_unstable); 1165 1166 static void __init check_system_tsc_reliable(void) 1167 { 1168 #if defined(CONFIG_MGEODEGX1) || defined(CONFIG_MGEODE_LX) || defined(CONFIG_X86_GENERIC) 1169 if (is_geode_lx()) { 1170 /* RTSC counts during suspend */ 1171 #define RTSC_SUSP 0x100 1172 unsigned long res_low, res_high; 1173 1174 rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high); 1175 /* Geode_LX - the OLPC CPU has a very reliable TSC */ 1176 if (res_low & RTSC_SUSP) 1177 tsc_clocksource_reliable = 1; 1178 } 1179 #endif 1180 if (boot_cpu_has(X86_FEATURE_TSC_RELIABLE)) 1181 tsc_clocksource_reliable = 1; 1182 } 1183 1184 /* 1185 * Make an educated guess if the TSC is trustworthy and synchronized 1186 * over all CPUs. 1187 */ 1188 int unsynchronized_tsc(void) 1189 { 1190 if (!boot_cpu_has(X86_FEATURE_TSC) || tsc_unstable) 1191 return 1; 1192 1193 #ifdef CONFIG_SMP 1194 if (apic_is_clustered_box()) 1195 return 1; 1196 #endif 1197 1198 if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) 1199 return 0; 1200 1201 if (tsc_clocksource_reliable) 1202 return 0; 1203 /* 1204 * Intel systems are normally all synchronized. 1205 * Exceptions must mark TSC as unstable: 1206 */ 1207 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { 1208 /* assume multi socket systems are not synchronized: */ 1209 if (num_possible_cpus() > 1) 1210 return 1; 1211 } 1212 1213 return 0; 1214 } 1215 1216 /* 1217 * Convert ART to TSC given numerator/denominator found in detect_art() 1218 */ 1219 struct system_counterval_t convert_art_to_tsc(u64 art) 1220 { 1221 u64 tmp, res, rem; 1222 1223 rem = do_div(art, art_to_tsc_denominator); 1224 1225 res = art * art_to_tsc_numerator; 1226 tmp = rem * art_to_tsc_numerator; 1227 1228 do_div(tmp, art_to_tsc_denominator); 1229 res += tmp + art_to_tsc_offset; 1230 1231 return (struct system_counterval_t) {.cs = art_related_clocksource, 1232 .cycles = res}; 1233 } 1234 EXPORT_SYMBOL(convert_art_to_tsc); 1235 1236 static void tsc_refine_calibration_work(struct work_struct *work); 1237 static DECLARE_DELAYED_WORK(tsc_irqwork, tsc_refine_calibration_work); 1238 /** 1239 * tsc_refine_calibration_work - Further refine tsc freq calibration 1240 * @work - ignored. 1241 * 1242 * This functions uses delayed work over a period of a 1243 * second to further refine the TSC freq value. Since this is 1244 * timer based, instead of loop based, we don't block the boot 1245 * process while this longer calibration is done. 1246 * 1247 * If there are any calibration anomalies (too many SMIs, etc), 1248 * or the refined calibration is off by 1% of the fast early 1249 * calibration, we throw out the new calibration and use the 1250 * early calibration. 1251 */ 1252 static void tsc_refine_calibration_work(struct work_struct *work) 1253 { 1254 static u64 tsc_start = -1, ref_start; 1255 static int hpet; 1256 u64 tsc_stop, ref_stop, delta; 1257 unsigned long freq; 1258 1259 /* Don't bother refining TSC on unstable systems */ 1260 if (check_tsc_unstable()) 1261 goto out; 1262 1263 /* 1264 * Since the work is started early in boot, we may be 1265 * delayed the first time we expire. So set the workqueue 1266 * again once we know timers are working. 1267 */ 1268 if (tsc_start == -1) { 1269 /* 1270 * Only set hpet once, to avoid mixing hardware 1271 * if the hpet becomes enabled later. 1272 */ 1273 hpet = is_hpet_enabled(); 1274 schedule_delayed_work(&tsc_irqwork, HZ); 1275 tsc_start = tsc_read_refs(&ref_start, hpet); 1276 return; 1277 } 1278 1279 tsc_stop = tsc_read_refs(&ref_stop, hpet); 1280 1281 /* hpet or pmtimer available ? */ 1282 if (ref_start == ref_stop) 1283 goto out; 1284 1285 /* Check, whether the sampling was disturbed by an SMI */ 1286 if (tsc_start == ULLONG_MAX || tsc_stop == ULLONG_MAX) 1287 goto out; 1288 1289 delta = tsc_stop - tsc_start; 1290 delta *= 1000000LL; 1291 if (hpet) 1292 freq = calc_hpet_ref(delta, ref_start, ref_stop); 1293 else 1294 freq = calc_pmtimer_ref(delta, ref_start, ref_stop); 1295 1296 /* Make sure we're within 1% */ 1297 if (abs(tsc_khz - freq) > tsc_khz/100) 1298 goto out; 1299 1300 tsc_khz = freq; 1301 pr_info("Refined TSC clocksource calibration: %lu.%03lu MHz\n", 1302 (unsigned long)tsc_khz / 1000, 1303 (unsigned long)tsc_khz % 1000); 1304 1305 /* Inform the TSC deadline clockevent devices about the recalibration */ 1306 lapic_update_tsc_freq(); 1307 1308 out: 1309 if (boot_cpu_has(X86_FEATURE_ART)) 1310 art_related_clocksource = &clocksource_tsc; 1311 clocksource_register_khz(&clocksource_tsc, tsc_khz); 1312 } 1313 1314 1315 static int __init init_tsc_clocksource(void) 1316 { 1317 if (!boot_cpu_has(X86_FEATURE_TSC) || tsc_disabled > 0 || !tsc_khz) 1318 return 0; 1319 1320 if (tsc_clocksource_reliable) 1321 clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY; 1322 /* lower the rating if we already know its unstable: */ 1323 if (check_tsc_unstable()) { 1324 clocksource_tsc.rating = 0; 1325 clocksource_tsc.flags &= ~CLOCK_SOURCE_IS_CONTINUOUS; 1326 } 1327 1328 if (boot_cpu_has(X86_FEATURE_NONSTOP_TSC_S3)) 1329 clocksource_tsc.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP; 1330 1331 /* 1332 * When TSC frequency is known (retrieved via MSR or CPUID), we skip 1333 * the refined calibration and directly register it as a clocksource. 1334 */ 1335 if (boot_cpu_has(X86_FEATURE_TSC_KNOWN_FREQ)) { 1336 if (boot_cpu_has(X86_FEATURE_ART)) 1337 art_related_clocksource = &clocksource_tsc; 1338 clocksource_register_khz(&clocksource_tsc, tsc_khz); 1339 return 0; 1340 } 1341 1342 schedule_delayed_work(&tsc_irqwork, 0); 1343 return 0; 1344 } 1345 /* 1346 * We use device_initcall here, to ensure we run after the hpet 1347 * is fully initialized, which may occur at fs_initcall time. 1348 */ 1349 device_initcall(init_tsc_clocksource); 1350 1351 void __init tsc_init(void) 1352 { 1353 u64 lpj; 1354 int cpu; 1355 1356 if (!boot_cpu_has(X86_FEATURE_TSC)) { 1357 setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER); 1358 return; 1359 } 1360 1361 cpu_khz = x86_platform.calibrate_cpu(); 1362 tsc_khz = x86_platform.calibrate_tsc(); 1363 1364 /* 1365 * Trust non-zero tsc_khz as authorative, 1366 * and use it to sanity check cpu_khz, 1367 * which will be off if system timer is off. 1368 */ 1369 if (tsc_khz == 0) 1370 tsc_khz = cpu_khz; 1371 else if (abs(cpu_khz - tsc_khz) * 10 > tsc_khz) 1372 cpu_khz = tsc_khz; 1373 1374 if (!tsc_khz) { 1375 mark_tsc_unstable("could not calculate TSC khz"); 1376 setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER); 1377 return; 1378 } 1379 1380 pr_info("Detected %lu.%03lu MHz processor\n", 1381 (unsigned long)cpu_khz / 1000, 1382 (unsigned long)cpu_khz % 1000); 1383 1384 /* Sanitize TSC ADJUST before cyc2ns gets initialized */ 1385 tsc_store_and_check_tsc_adjust(true); 1386 1387 /* 1388 * Secondary CPUs do not run through tsc_init(), so set up 1389 * all the scale factors for all CPUs, assuming the same 1390 * speed as the bootup CPU. (cpufreq notifiers will fix this 1391 * up if their speed diverges) 1392 */ 1393 for_each_possible_cpu(cpu) { 1394 cyc2ns_init(cpu); 1395 set_cyc2ns_scale(tsc_khz, cpu); 1396 } 1397 1398 if (tsc_disabled > 0) 1399 return; 1400 1401 /* now allow native_sched_clock() to use rdtsc */ 1402 1403 tsc_disabled = 0; 1404 static_branch_enable(&__use_tsc); 1405 1406 if (!no_sched_irq_time) 1407 enable_sched_clock_irqtime(); 1408 1409 lpj = ((u64)tsc_khz * 1000); 1410 do_div(lpj, HZ); 1411 lpj_fine = lpj; 1412 1413 use_tsc_delay(); 1414 1415 if (unsynchronized_tsc()) 1416 mark_tsc_unstable("TSCs unsynchronized"); 1417 1418 check_system_tsc_reliable(); 1419 1420 detect_art(); 1421 } 1422 1423 #ifdef CONFIG_SMP 1424 /* 1425 * If we have a constant TSC and are using the TSC for the delay loop, 1426 * we can skip clock calibration if another cpu in the same socket has already 1427 * been calibrated. This assumes that CONSTANT_TSC applies to all 1428 * cpus in the socket - this should be a safe assumption. 1429 */ 1430 unsigned long calibrate_delay_is_known(void) 1431 { 1432 int sibling, cpu = smp_processor_id(); 1433 struct cpumask *mask = topology_core_cpumask(cpu); 1434 1435 if (!tsc_disabled && !cpu_has(&cpu_data(cpu), X86_FEATURE_CONSTANT_TSC)) 1436 return 0; 1437 1438 if (!mask) 1439 return 0; 1440 1441 sibling = cpumask_any_but(mask, cpu); 1442 if (sibling < nr_cpu_ids) 1443 return cpu_data(sibling).loops_per_jiffy; 1444 return 0; 1445 } 1446 #endif 1447