1 /* 2 * check TSC synchronization. 3 * 4 * Copyright (C) 2006, Red Hat, Inc., Ingo Molnar 5 * 6 * We check whether all boot CPUs have their TSC's synchronized, 7 * print a warning if not and turn off the TSC clock-source. 8 * 9 * The warp-check is point-to-point between two CPUs, the CPU 10 * initiating the bootup is the 'source CPU', the freshly booting 11 * CPU is the 'target CPU'. 12 * 13 * Only two CPUs may participate - they can enter in any order. 14 * ( The serial nature of the boot logic and the CPU hotplug lock 15 * protects against more than 2 CPUs entering this code. ) 16 */ 17 #include <linux/spinlock.h> 18 #include <linux/kernel.h> 19 #include <linux/smp.h> 20 #include <linux/nmi.h> 21 #include <asm/tsc.h> 22 23 /* 24 * Entry/exit counters that make sure that both CPUs 25 * run the measurement code at once: 26 */ 27 static atomic_t start_count; 28 static atomic_t stop_count; 29 30 /* 31 * We use a raw spinlock in this exceptional case, because 32 * we want to have the fastest, inlined, non-debug version 33 * of a critical section, to be able to prove TSC time-warps: 34 */ 35 static arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED; 36 37 static cycles_t last_tsc; 38 static cycles_t max_warp; 39 static int nr_warps; 40 static int random_warps; 41 42 /* 43 * TSC-warp measurement loop running on both CPUs. This is not called 44 * if there is no TSC. 45 */ 46 static void check_tsc_warp(unsigned int timeout) 47 { 48 cycles_t start, now, prev, end; 49 int i, cur_warps = 0; 50 51 start = rdtsc_ordered(); 52 /* 53 * The measurement runs for 'timeout' msecs: 54 */ 55 end = start + (cycles_t) tsc_khz * timeout; 56 now = start; 57 58 for (i = 0; ; i++) { 59 /* 60 * We take the global lock, measure TSC, save the 61 * previous TSC that was measured (possibly on 62 * another CPU) and update the previous TSC timestamp. 63 */ 64 arch_spin_lock(&sync_lock); 65 prev = last_tsc; 66 now = rdtsc_ordered(); 67 last_tsc = now; 68 arch_spin_unlock(&sync_lock); 69 70 /* 71 * Be nice every now and then (and also check whether 72 * measurement is done [we also insert a 10 million 73 * loops safety exit, so we dont lock up in case the 74 * TSC readout is totally broken]): 75 */ 76 if (unlikely(!(i & 7))) { 77 if (now > end || i > 10000000) 78 break; 79 cpu_relax(); 80 touch_nmi_watchdog(); 81 } 82 /* 83 * Outside the critical section we can now see whether 84 * we saw a time-warp of the TSC going backwards: 85 */ 86 if (unlikely(prev > now)) { 87 arch_spin_lock(&sync_lock); 88 max_warp = max(max_warp, prev - now); 89 /* 90 * Check whether this bounces back and forth. Only 91 * one CPU should observe time going backwards. 92 */ 93 if (cur_warps != nr_warps) 94 random_warps++; 95 nr_warps++; 96 cur_warps = nr_warps; 97 arch_spin_unlock(&sync_lock); 98 } 99 } 100 WARN(!(now-start), 101 "Warning: zero tsc calibration delta: %Ld [max: %Ld]\n", 102 now-start, end-start); 103 } 104 105 /* 106 * If the target CPU coming online doesn't have any of its core-siblings 107 * online, a timeout of 20msec will be used for the TSC-warp measurement 108 * loop. Otherwise a smaller timeout of 2msec will be used, as we have some 109 * information about this socket already (and this information grows as we 110 * have more and more logical-siblings in that socket). 111 * 112 * Ideally we should be able to skip the TSC sync check on the other 113 * core-siblings, if the first logical CPU in a socket passed the sync test. 114 * But as the TSC is per-logical CPU and can potentially be modified wrongly 115 * by the bios, TSC sync test for smaller duration should be able 116 * to catch such errors. Also this will catch the condition where all the 117 * cores in the socket doesn't get reset at the same time. 118 */ 119 static inline unsigned int loop_timeout(int cpu) 120 { 121 return (cpumask_weight(topology_core_cpumask(cpu)) > 1) ? 2 : 20; 122 } 123 124 /* 125 * Source CPU calls into this - it waits for the freshly booted 126 * target CPU to arrive and then starts the measurement: 127 */ 128 void check_tsc_sync_source(int cpu) 129 { 130 int cpus = 2; 131 132 /* 133 * No need to check if we already know that the TSC is not 134 * synchronized or if we have no TSC. 135 */ 136 if (unsynchronized_tsc()) 137 return; 138 139 if (tsc_clocksource_reliable) { 140 if (cpu == (nr_cpu_ids-1) || system_state != SYSTEM_BOOTING) 141 pr_info( 142 "Skipped synchronization checks as TSC is reliable.\n"); 143 return; 144 } 145 146 /* 147 * Reset it - in case this is a second bootup: 148 */ 149 atomic_set(&stop_count, 0); 150 151 /* 152 * Wait for the target to arrive: 153 */ 154 while (atomic_read(&start_count) != cpus-1) 155 cpu_relax(); 156 /* 157 * Trigger the target to continue into the measurement too: 158 */ 159 atomic_inc(&start_count); 160 161 check_tsc_warp(loop_timeout(cpu)); 162 163 while (atomic_read(&stop_count) != cpus-1) 164 cpu_relax(); 165 166 if (nr_warps) { 167 pr_warning("TSC synchronization [CPU#%d -> CPU#%d]:\n", 168 smp_processor_id(), cpu); 169 pr_warning("Measured %Ld cycles TSC warp between CPUs, " 170 "turning off TSC clock.\n", max_warp); 171 if (random_warps) 172 pr_warning("TSC warped randomly between CPUs\n"); 173 mark_tsc_unstable("check_tsc_sync_source failed"); 174 } else { 175 pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n", 176 smp_processor_id(), cpu); 177 } 178 179 /* 180 * Reset it - just in case we boot another CPU later: 181 */ 182 atomic_set(&start_count, 0); 183 random_warps = 0; 184 nr_warps = 0; 185 max_warp = 0; 186 last_tsc = 0; 187 188 /* 189 * Let the target continue with the bootup: 190 */ 191 atomic_inc(&stop_count); 192 } 193 194 /* 195 * Freshly booted CPUs call into this: 196 */ 197 void check_tsc_sync_target(void) 198 { 199 int cpus = 2; 200 201 /* Also aborts if there is no TSC. */ 202 if (unsynchronized_tsc() || tsc_clocksource_reliable) 203 return; 204 205 /* 206 * Register this CPU's participation and wait for the 207 * source CPU to start the measurement: 208 */ 209 atomic_inc(&start_count); 210 while (atomic_read(&start_count) != cpus) 211 cpu_relax(); 212 213 check_tsc_warp(loop_timeout(smp_processor_id())); 214 215 /* 216 * Ok, we are done: 217 */ 218 atomic_inc(&stop_count); 219 220 /* 221 * Wait for the source CPU to print stuff: 222 */ 223 while (atomic_read(&stop_count) != cpus) 224 cpu_relax(); 225 } 226