1 /* 2 * check TSC synchronization. 3 * 4 * Copyright (C) 2006, Red Hat, Inc., Ingo Molnar 5 * 6 * We check whether all boot CPUs have their TSC's synchronized, 7 * print a warning if not and turn off the TSC clock-source. 8 * 9 * The warp-check is point-to-point between two CPUs, the CPU 10 * initiating the bootup is the 'source CPU', the freshly booting 11 * CPU is the 'target CPU'. 12 * 13 * Only two CPUs may participate - they can enter in any order. 14 * ( The serial nature of the boot logic and the CPU hotplug lock 15 * protects against more than 2 CPUs entering this code. ) 16 */ 17 #include <linux/spinlock.h> 18 #include <linux/kernel.h> 19 #include <linux/smp.h> 20 #include <linux/nmi.h> 21 #include <asm/tsc.h> 22 23 /* 24 * Entry/exit counters that make sure that both CPUs 25 * run the measurement code at once: 26 */ 27 static atomic_t start_count; 28 static atomic_t stop_count; 29 30 /* 31 * We use a raw spinlock in this exceptional case, because 32 * we want to have the fastest, inlined, non-debug version 33 * of a critical section, to be able to prove TSC time-warps: 34 */ 35 static arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED; 36 37 static cycles_t last_tsc; 38 static cycles_t max_warp; 39 static int nr_warps; 40 41 /* 42 * TSC-warp measurement loop running on both CPUs. This is not called 43 * if there is no TSC. 44 */ 45 static void check_tsc_warp(unsigned int timeout) 46 { 47 cycles_t start, now, prev, end; 48 int i; 49 50 start = rdtsc_ordered(); 51 /* 52 * The measurement runs for 'timeout' msecs: 53 */ 54 end = start + (cycles_t) tsc_khz * timeout; 55 now = start; 56 57 for (i = 0; ; i++) { 58 /* 59 * We take the global lock, measure TSC, save the 60 * previous TSC that was measured (possibly on 61 * another CPU) and update the previous TSC timestamp. 62 */ 63 arch_spin_lock(&sync_lock); 64 prev = last_tsc; 65 now = rdtsc_ordered(); 66 last_tsc = now; 67 arch_spin_unlock(&sync_lock); 68 69 /* 70 * Be nice every now and then (and also check whether 71 * measurement is done [we also insert a 10 million 72 * loops safety exit, so we dont lock up in case the 73 * TSC readout is totally broken]): 74 */ 75 if (unlikely(!(i & 7))) { 76 if (now > end || i > 10000000) 77 break; 78 cpu_relax(); 79 touch_nmi_watchdog(); 80 } 81 /* 82 * Outside the critical section we can now see whether 83 * we saw a time-warp of the TSC going backwards: 84 */ 85 if (unlikely(prev > now)) { 86 arch_spin_lock(&sync_lock); 87 max_warp = max(max_warp, prev - now); 88 nr_warps++; 89 arch_spin_unlock(&sync_lock); 90 } 91 } 92 WARN(!(now-start), 93 "Warning: zero tsc calibration delta: %Ld [max: %Ld]\n", 94 now-start, end-start); 95 } 96 97 /* 98 * If the target CPU coming online doesn't have any of its core-siblings 99 * online, a timeout of 20msec will be used for the TSC-warp measurement 100 * loop. Otherwise a smaller timeout of 2msec will be used, as we have some 101 * information about this socket already (and this information grows as we 102 * have more and more logical-siblings in that socket). 103 * 104 * Ideally we should be able to skip the TSC sync check on the other 105 * core-siblings, if the first logical CPU in a socket passed the sync test. 106 * But as the TSC is per-logical CPU and can potentially be modified wrongly 107 * by the bios, TSC sync test for smaller duration should be able 108 * to catch such errors. Also this will catch the condition where all the 109 * cores in the socket doesn't get reset at the same time. 110 */ 111 static inline unsigned int loop_timeout(int cpu) 112 { 113 return (cpumask_weight(topology_core_cpumask(cpu)) > 1) ? 2 : 20; 114 } 115 116 /* 117 * Source CPU calls into this - it waits for the freshly booted 118 * target CPU to arrive and then starts the measurement: 119 */ 120 void check_tsc_sync_source(int cpu) 121 { 122 int cpus = 2; 123 124 /* 125 * No need to check if we already know that the TSC is not 126 * synchronized or if we have no TSC. 127 */ 128 if (unsynchronized_tsc()) 129 return; 130 131 if (tsc_clocksource_reliable) { 132 if (cpu == (nr_cpu_ids-1) || system_state != SYSTEM_BOOTING) 133 pr_info( 134 "Skipped synchronization checks as TSC is reliable.\n"); 135 return; 136 } 137 138 /* 139 * Reset it - in case this is a second bootup: 140 */ 141 atomic_set(&stop_count, 0); 142 143 /* 144 * Wait for the target to arrive: 145 */ 146 while (atomic_read(&start_count) != cpus-1) 147 cpu_relax(); 148 /* 149 * Trigger the target to continue into the measurement too: 150 */ 151 atomic_inc(&start_count); 152 153 check_tsc_warp(loop_timeout(cpu)); 154 155 while (atomic_read(&stop_count) != cpus-1) 156 cpu_relax(); 157 158 if (nr_warps) { 159 pr_warning("TSC synchronization [CPU#%d -> CPU#%d]:\n", 160 smp_processor_id(), cpu); 161 pr_warning("Measured %Ld cycles TSC warp between CPUs, " 162 "turning off TSC clock.\n", max_warp); 163 mark_tsc_unstable("check_tsc_sync_source failed"); 164 } else { 165 pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n", 166 smp_processor_id(), cpu); 167 } 168 169 /* 170 * Reset it - just in case we boot another CPU later: 171 */ 172 atomic_set(&start_count, 0); 173 nr_warps = 0; 174 max_warp = 0; 175 last_tsc = 0; 176 177 /* 178 * Let the target continue with the bootup: 179 */ 180 atomic_inc(&stop_count); 181 } 182 183 /* 184 * Freshly booted CPUs call into this: 185 */ 186 void check_tsc_sync_target(void) 187 { 188 int cpus = 2; 189 190 /* Also aborts if there is no TSC. */ 191 if (unsynchronized_tsc() || tsc_clocksource_reliable) 192 return; 193 194 /* 195 * Register this CPU's participation and wait for the 196 * source CPU to start the measurement: 197 */ 198 atomic_inc(&start_count); 199 while (atomic_read(&start_count) != cpus) 200 cpu_relax(); 201 202 check_tsc_warp(loop_timeout(smp_processor_id())); 203 204 /* 205 * Ok, we are done: 206 */ 207 atomic_inc(&stop_count); 208 209 /* 210 * Wait for the source CPU to print stuff: 211 */ 212 while (atomic_read(&stop_count) != cpus) 213 cpu_relax(); 214 } 215