1 /* 2 * check TSC synchronization. 3 * 4 * Copyright (C) 2006, Red Hat, Inc., Ingo Molnar 5 * 6 * We check whether all boot CPUs have their TSC's synchronized, 7 * print a warning if not and turn off the TSC clock-source. 8 * 9 * The warp-check is point-to-point between two CPUs, the CPU 10 * initiating the bootup is the 'source CPU', the freshly booting 11 * CPU is the 'target CPU'. 12 * 13 * Only two CPUs may participate - they can enter in any order. 14 * ( The serial nature of the boot logic and the CPU hotplug lock 15 * protects against more than 2 CPUs entering this code. ) 16 */ 17 #include <linux/spinlock.h> 18 #include <linux/kernel.h> 19 #include <linux/init.h> 20 #include <linux/smp.h> 21 #include <linux/nmi.h> 22 #include <asm/tsc.h> 23 24 /* 25 * Entry/exit counters that make sure that both CPUs 26 * run the measurement code at once: 27 */ 28 static atomic_t start_count; 29 static atomic_t stop_count; 30 31 /* 32 * We use a raw spinlock in this exceptional case, because 33 * we want to have the fastest, inlined, non-debug version 34 * of a critical section, to be able to prove TSC time-warps: 35 */ 36 static arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED; 37 38 static cycles_t last_tsc; 39 static cycles_t max_warp; 40 static int nr_warps; 41 42 /* 43 * TSC-warp measurement loop running on both CPUs: 44 */ 45 static void check_tsc_warp(unsigned int timeout) 46 { 47 cycles_t start, now, prev, end; 48 int i; 49 50 rdtsc_barrier(); 51 start = get_cycles(); 52 rdtsc_barrier(); 53 /* 54 * The measurement runs for 'timeout' msecs: 55 */ 56 end = start + (cycles_t) tsc_khz * timeout; 57 now = start; 58 59 for (i = 0; ; i++) { 60 /* 61 * We take the global lock, measure TSC, save the 62 * previous TSC that was measured (possibly on 63 * another CPU) and update the previous TSC timestamp. 64 */ 65 arch_spin_lock(&sync_lock); 66 prev = last_tsc; 67 rdtsc_barrier(); 68 now = get_cycles(); 69 rdtsc_barrier(); 70 last_tsc = now; 71 arch_spin_unlock(&sync_lock); 72 73 /* 74 * Be nice every now and then (and also check whether 75 * measurement is done [we also insert a 10 million 76 * loops safety exit, so we dont lock up in case the 77 * TSC readout is totally broken]): 78 */ 79 if (unlikely(!(i & 7))) { 80 if (now > end || i > 10000000) 81 break; 82 cpu_relax(); 83 touch_nmi_watchdog(); 84 } 85 /* 86 * Outside the critical section we can now see whether 87 * we saw a time-warp of the TSC going backwards: 88 */ 89 if (unlikely(prev > now)) { 90 arch_spin_lock(&sync_lock); 91 max_warp = max(max_warp, prev - now); 92 nr_warps++; 93 arch_spin_unlock(&sync_lock); 94 } 95 } 96 WARN(!(now-start), 97 "Warning: zero tsc calibration delta: %Ld [max: %Ld]\n", 98 now-start, end-start); 99 } 100 101 /* 102 * If the target CPU coming online doesn't have any of its core-siblings 103 * online, a timeout of 20msec will be used for the TSC-warp measurement 104 * loop. Otherwise a smaller timeout of 2msec will be used, as we have some 105 * information about this socket already (and this information grows as we 106 * have more and more logical-siblings in that socket). 107 * 108 * Ideally we should be able to skip the TSC sync check on the other 109 * core-siblings, if the first logical CPU in a socket passed the sync test. 110 * But as the TSC is per-logical CPU and can potentially be modified wrongly 111 * by the bios, TSC sync test for smaller duration should be able 112 * to catch such errors. Also this will catch the condition where all the 113 * cores in the socket doesn't get reset at the same time. 114 */ 115 static inline unsigned int loop_timeout(int cpu) 116 { 117 return (cpumask_weight(cpu_core_mask(cpu)) > 1) ? 2 : 20; 118 } 119 120 /* 121 * Source CPU calls into this - it waits for the freshly booted 122 * target CPU to arrive and then starts the measurement: 123 */ 124 void check_tsc_sync_source(int cpu) 125 { 126 int cpus = 2; 127 128 /* 129 * No need to check if we already know that the TSC is not 130 * synchronized: 131 */ 132 if (unsynchronized_tsc()) 133 return; 134 135 if (tsc_clocksource_reliable) { 136 if (cpu == (nr_cpu_ids-1) || system_state != SYSTEM_BOOTING) 137 pr_info( 138 "Skipped synchronization checks as TSC is reliable.\n"); 139 return; 140 } 141 142 /* 143 * Reset it - in case this is a second bootup: 144 */ 145 atomic_set(&stop_count, 0); 146 147 /* 148 * Wait for the target to arrive: 149 */ 150 while (atomic_read(&start_count) != cpus-1) 151 cpu_relax(); 152 /* 153 * Trigger the target to continue into the measurement too: 154 */ 155 atomic_inc(&start_count); 156 157 check_tsc_warp(loop_timeout(cpu)); 158 159 while (atomic_read(&stop_count) != cpus-1) 160 cpu_relax(); 161 162 if (nr_warps) { 163 pr_warning("TSC synchronization [CPU#%d -> CPU#%d]:\n", 164 smp_processor_id(), cpu); 165 pr_warning("Measured %Ld cycles TSC warp between CPUs, " 166 "turning off TSC clock.\n", max_warp); 167 mark_tsc_unstable("check_tsc_sync_source failed"); 168 } else { 169 pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n", 170 smp_processor_id(), cpu); 171 } 172 173 /* 174 * Reset it - just in case we boot another CPU later: 175 */ 176 atomic_set(&start_count, 0); 177 nr_warps = 0; 178 max_warp = 0; 179 last_tsc = 0; 180 181 /* 182 * Let the target continue with the bootup: 183 */ 184 atomic_inc(&stop_count); 185 } 186 187 /* 188 * Freshly booted CPUs call into this: 189 */ 190 void check_tsc_sync_target(void) 191 { 192 int cpus = 2; 193 194 if (unsynchronized_tsc() || tsc_clocksource_reliable) 195 return; 196 197 /* 198 * Register this CPU's participation and wait for the 199 * source CPU to start the measurement: 200 */ 201 atomic_inc(&start_count); 202 while (atomic_read(&start_count) != cpus) 203 cpu_relax(); 204 205 check_tsc_warp(loop_timeout(smp_processor_id())); 206 207 /* 208 * Ok, we are done: 209 */ 210 atomic_inc(&stop_count); 211 212 /* 213 * Wait for the source CPU to print stuff: 214 */ 215 while (atomic_read(&stop_count) != cpus) 216 cpu_relax(); 217 } 218