1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * SMP Support 4 * 5 * Copyright (C) 1999 Walt Drummond <drummond@valinux.com> 6 * Copyright (C) 1999, 2001, 2003 David Mosberger-Tang <davidm@hpl.hp.com> 7 * 8 * Lots of stuff stolen from arch/alpha/kernel/smp.c 9 * 10 * 01/05/16 Rohit Seth <rohit.seth@intel.com> IA64-SMP functions. Reorganized 11 * the existing code (on the lines of x86 port). 12 * 00/09/11 David Mosberger <davidm@hpl.hp.com> Do loops_per_jiffy 13 * calibration on each CPU. 14 * 00/08/23 Asit Mallick <asit.k.mallick@intel.com> fixed logical processor id 15 * 00/03/31 Rohit Seth <rohit.seth@intel.com> Fixes for Bootstrap Processor 16 * & cpu_online_map now gets done here (instead of setup.c) 17 * 99/10/05 davidm Update to bring it in sync with new command-line processing 18 * scheme. 19 * 10/13/00 Goutham Rao <goutham.rao@intel.com> Updated smp_call_function and 20 * smp_call_function_single to resend IPI on timeouts 21 */ 22 #include <linux/module.h> 23 #include <linux/kernel.h> 24 #include <linux/sched.h> 25 #include <linux/init.h> 26 #include <linux/interrupt.h> 27 #include <linux/smp.h> 28 #include <linux/kernel_stat.h> 29 #include <linux/mm.h> 30 #include <linux/cache.h> 31 #include <linux/delay.h> 32 #include <linux/efi.h> 33 #include <linux/bitops.h> 34 #include <linux/kexec.h> 35 36 #include <linux/atomic.h> 37 #include <asm/current.h> 38 #include <asm/delay.h> 39 #include <asm/io.h> 40 #include <asm/irq.h> 41 #include <asm/page.h> 42 #include <asm/processor.h> 43 #include <asm/ptrace.h> 44 #include <asm/sal.h> 45 #include <asm/tlbflush.h> 46 #include <asm/unistd.h> 47 #include <asm/mca.h> 48 #include <asm/xtp.h> 49 50 /* 51 * Note: alignment of 4 entries/cacheline was empirically determined 52 * to be a good tradeoff between hot cachelines & spreading the array 53 * across too many cacheline. 54 */ 55 static struct local_tlb_flush_counts { 56 unsigned int count; 57 } __attribute__((__aligned__(32))) local_tlb_flush_counts[NR_CPUS]; 58 59 static DEFINE_PER_CPU_SHARED_ALIGNED(unsigned short [NR_CPUS], 60 shadow_flush_counts); 61 62 #define IPI_CALL_FUNC 0 63 #define IPI_CPU_STOP 1 64 #define IPI_CALL_FUNC_SINGLE 2 65 #define IPI_KDUMP_CPU_STOP 3 66 67 /* This needs to be cacheline aligned because it is written to by *other* CPUs. */ 68 static DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, ipi_operation); 69 70 extern void cpu_halt (void); 71 72 static void 73 stop_this_cpu(void) 74 { 75 /* 76 * Remove this CPU: 77 */ 78 set_cpu_online(smp_processor_id(), false); 79 max_xtp(); 80 local_irq_disable(); 81 cpu_halt(); 82 } 83 84 void 85 cpu_die(void) 86 { 87 max_xtp(); 88 local_irq_disable(); 89 cpu_halt(); 90 /* Should never be here */ 91 BUG(); 92 for (;;); 93 } 94 95 irqreturn_t 96 handle_IPI (int irq, void *dev_id) 97 { 98 int this_cpu = get_cpu(); 99 unsigned long *pending_ipis = &__ia64_per_cpu_var(ipi_operation); 100 unsigned long ops; 101 102 mb(); /* Order interrupt and bit testing. */ 103 while ((ops = xchg(pending_ipis, 0)) != 0) { 104 mb(); /* Order bit clearing and data access. */ 105 do { 106 unsigned long which; 107 108 which = ffz(~ops); 109 ops &= ~(1 << which); 110 111 switch (which) { 112 case IPI_CPU_STOP: 113 stop_this_cpu(); 114 break; 115 case IPI_CALL_FUNC: 116 generic_smp_call_function_interrupt(); 117 break; 118 case IPI_CALL_FUNC_SINGLE: 119 generic_smp_call_function_single_interrupt(); 120 break; 121 #ifdef CONFIG_KEXEC 122 case IPI_KDUMP_CPU_STOP: 123 unw_init_running(kdump_cpu_freeze, NULL); 124 break; 125 #endif 126 default: 127 printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n", 128 this_cpu, which); 129 break; 130 } 131 } while (ops); 132 mb(); /* Order data access and bit testing. */ 133 } 134 put_cpu(); 135 return IRQ_HANDLED; 136 } 137 138 139 140 /* 141 * Called with preemption disabled. 142 */ 143 static inline void 144 send_IPI_single (int dest_cpu, int op) 145 { 146 set_bit(op, &per_cpu(ipi_operation, dest_cpu)); 147 ia64_send_ipi(dest_cpu, IA64_IPI_VECTOR, IA64_IPI_DM_INT, 0); 148 } 149 150 /* 151 * Called with preemption disabled. 152 */ 153 static inline void 154 send_IPI_allbutself (int op) 155 { 156 unsigned int i; 157 158 for_each_online_cpu(i) { 159 if (i != smp_processor_id()) 160 send_IPI_single(i, op); 161 } 162 } 163 164 /* 165 * Called with preemption disabled. 166 */ 167 static inline void 168 send_IPI_mask(const struct cpumask *mask, int op) 169 { 170 unsigned int cpu; 171 172 for_each_cpu(cpu, mask) { 173 send_IPI_single(cpu, op); 174 } 175 } 176 177 /* 178 * Called with preemption disabled. 179 */ 180 static inline void 181 send_IPI_all (int op) 182 { 183 int i; 184 185 for_each_online_cpu(i) { 186 send_IPI_single(i, op); 187 } 188 } 189 190 /* 191 * Called with preemption disabled. 192 */ 193 static inline void 194 send_IPI_self (int op) 195 { 196 send_IPI_single(smp_processor_id(), op); 197 } 198 199 #ifdef CONFIG_KEXEC 200 void 201 kdump_smp_send_stop(void) 202 { 203 send_IPI_allbutself(IPI_KDUMP_CPU_STOP); 204 } 205 206 void 207 kdump_smp_send_init(void) 208 { 209 unsigned int cpu, self_cpu; 210 self_cpu = smp_processor_id(); 211 for_each_online_cpu(cpu) { 212 if (cpu != self_cpu) { 213 if(kdump_status[cpu] == 0) 214 ia64_send_ipi(cpu, 0, IA64_IPI_DM_INIT, 0); 215 } 216 } 217 } 218 #endif 219 /* 220 * Called with preemption disabled. 221 */ 222 void 223 arch_smp_send_reschedule (int cpu) 224 { 225 ia64_send_ipi(cpu, IA64_IPI_RESCHEDULE, IA64_IPI_DM_INT, 0); 226 } 227 EXPORT_SYMBOL_GPL(arch_smp_send_reschedule); 228 229 /* 230 * Called with preemption disabled. 231 */ 232 static void 233 smp_send_local_flush_tlb (int cpu) 234 { 235 ia64_send_ipi(cpu, IA64_IPI_LOCAL_TLB_FLUSH, IA64_IPI_DM_INT, 0); 236 } 237 238 void 239 smp_local_flush_tlb(void) 240 { 241 /* 242 * Use atomic ops. Otherwise, the load/increment/store sequence from 243 * a "++" operation can have the line stolen between the load & store. 244 * The overhead of the atomic op in negligible in this case & offers 245 * significant benefit for the brief periods where lots of cpus 246 * are simultaneously flushing TLBs. 247 */ 248 ia64_fetchadd(1, &local_tlb_flush_counts[smp_processor_id()].count, acq); 249 local_flush_tlb_all(); 250 } 251 252 #define FLUSH_DELAY 5 /* Usec backoff to eliminate excessive cacheline bouncing */ 253 254 void 255 smp_flush_tlb_cpumask(cpumask_t xcpumask) 256 { 257 unsigned short *counts = __ia64_per_cpu_var(shadow_flush_counts); 258 cpumask_t cpumask = xcpumask; 259 int mycpu, cpu, flush_mycpu = 0; 260 261 preempt_disable(); 262 mycpu = smp_processor_id(); 263 264 for_each_cpu(cpu, &cpumask) 265 counts[cpu] = local_tlb_flush_counts[cpu].count & 0xffff; 266 267 mb(); 268 for_each_cpu(cpu, &cpumask) { 269 if (cpu == mycpu) 270 flush_mycpu = 1; 271 else 272 smp_send_local_flush_tlb(cpu); 273 } 274 275 if (flush_mycpu) 276 smp_local_flush_tlb(); 277 278 for_each_cpu(cpu, &cpumask) 279 while(counts[cpu] == (local_tlb_flush_counts[cpu].count & 0xffff)) 280 udelay(FLUSH_DELAY); 281 282 preempt_enable(); 283 } 284 285 void 286 smp_flush_tlb_all (void) 287 { 288 on_each_cpu((void (*)(void *))local_flush_tlb_all, NULL, 1); 289 } 290 291 void 292 smp_flush_tlb_mm (struct mm_struct *mm) 293 { 294 cpumask_var_t cpus; 295 preempt_disable(); 296 /* this happens for the common case of a single-threaded fork(): */ 297 if (likely(mm == current->active_mm && atomic_read(&mm->mm_users) == 1)) 298 { 299 local_finish_flush_tlb_mm(mm); 300 preempt_enable(); 301 return; 302 } 303 if (!alloc_cpumask_var(&cpus, GFP_ATOMIC)) { 304 smp_call_function((void (*)(void *))local_finish_flush_tlb_mm, 305 mm, 1); 306 } else { 307 cpumask_copy(cpus, mm_cpumask(mm)); 308 smp_call_function_many(cpus, 309 (void (*)(void *))local_finish_flush_tlb_mm, mm, 1); 310 free_cpumask_var(cpus); 311 } 312 local_irq_disable(); 313 local_finish_flush_tlb_mm(mm); 314 local_irq_enable(); 315 preempt_enable(); 316 } 317 318 void arch_send_call_function_single_ipi(int cpu) 319 { 320 send_IPI_single(cpu, IPI_CALL_FUNC_SINGLE); 321 } 322 323 void arch_send_call_function_ipi_mask(const struct cpumask *mask) 324 { 325 send_IPI_mask(mask, IPI_CALL_FUNC); 326 } 327 328 /* 329 * this function calls the 'stop' function on all other CPUs in the system. 330 */ 331 void 332 smp_send_stop (void) 333 { 334 send_IPI_allbutself(IPI_CPU_STOP); 335 } 336