1 /* 2 * Generic helpers for smp ipi calls 3 * 4 * (C) Jens Axboe <jens.axboe@oracle.com> 2008 5 */ 6 #include <linux/rcupdate.h> 7 #include <linux/rculist.h> 8 #include <linux/kernel.h> 9 #include <linux/module.h> 10 #include <linux/percpu.h> 11 #include <linux/init.h> 12 #include <linux/gfp.h> 13 #include <linux/smp.h> 14 #include <linux/cpu.h> 15 16 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS 17 static struct { 18 struct list_head queue; 19 raw_spinlock_t lock; 20 } call_function __cacheline_aligned_in_smp = 21 { 22 .queue = LIST_HEAD_INIT(call_function.queue), 23 .lock = __RAW_SPIN_LOCK_UNLOCKED(call_function.lock), 24 }; 25 26 enum { 27 CSD_FLAG_LOCK = 0x01, 28 }; 29 30 struct call_function_data { 31 struct call_single_data csd; 32 atomic_t refs; 33 cpumask_var_t cpumask; 34 }; 35 36 static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_function_data, cfd_data); 37 38 struct call_single_queue { 39 struct list_head list; 40 raw_spinlock_t lock; 41 }; 42 43 static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_queue, call_single_queue); 44 45 static int 46 hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu) 47 { 48 long cpu = (long)hcpu; 49 struct call_function_data *cfd = &per_cpu(cfd_data, cpu); 50 51 switch (action) { 52 case CPU_UP_PREPARE: 53 case CPU_UP_PREPARE_FROZEN: 54 if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL, 55 cpu_to_node(cpu))) 56 return notifier_from_errno(-ENOMEM); 57 break; 58 59 #ifdef CONFIG_HOTPLUG_CPU 60 case CPU_UP_CANCELED: 61 case CPU_UP_CANCELED_FROZEN: 62 63 case CPU_DEAD: 64 case CPU_DEAD_FROZEN: 65 free_cpumask_var(cfd->cpumask); 66 break; 67 #endif 68 }; 69 70 return NOTIFY_OK; 71 } 72 73 static struct notifier_block __cpuinitdata hotplug_cfd_notifier = { 74 .notifier_call = hotplug_cfd, 75 }; 76 77 static int __cpuinit init_call_single_data(void) 78 { 79 void *cpu = (void *)(long)smp_processor_id(); 80 int i; 81 82 for_each_possible_cpu(i) { 83 struct call_single_queue *q = &per_cpu(call_single_queue, i); 84 85 raw_spin_lock_init(&q->lock); 86 INIT_LIST_HEAD(&q->list); 87 } 88 89 hotplug_cfd(&hotplug_cfd_notifier, CPU_UP_PREPARE, cpu); 90 register_cpu_notifier(&hotplug_cfd_notifier); 91 92 return 0; 93 } 94 early_initcall(init_call_single_data); 95 96 /* 97 * csd_lock/csd_unlock used to serialize access to per-cpu csd resources 98 * 99 * For non-synchronous ipi calls the csd can still be in use by the 100 * previous function call. For multi-cpu calls its even more interesting 101 * as we'll have to ensure no other cpu is observing our csd. 102 */ 103 static void csd_lock_wait(struct call_single_data *data) 104 { 105 while (data->flags & CSD_FLAG_LOCK) 106 cpu_relax(); 107 } 108 109 static void csd_lock(struct call_single_data *data) 110 { 111 csd_lock_wait(data); 112 data->flags = CSD_FLAG_LOCK; 113 114 /* 115 * prevent CPU from reordering the above assignment 116 * to ->flags with any subsequent assignments to other 117 * fields of the specified call_single_data structure: 118 */ 119 smp_mb(); 120 } 121 122 static void csd_unlock(struct call_single_data *data) 123 { 124 WARN_ON(!(data->flags & CSD_FLAG_LOCK)); 125 126 /* 127 * ensure we're all done before releasing data: 128 */ 129 smp_mb(); 130 131 data->flags &= ~CSD_FLAG_LOCK; 132 } 133 134 /* 135 * Insert a previously allocated call_single_data element 136 * for execution on the given CPU. data must already have 137 * ->func, ->info, and ->flags set. 138 */ 139 static 140 void generic_exec_single(int cpu, struct call_single_data *data, int wait) 141 { 142 struct call_single_queue *dst = &per_cpu(call_single_queue, cpu); 143 unsigned long flags; 144 int ipi; 145 146 raw_spin_lock_irqsave(&dst->lock, flags); 147 ipi = list_empty(&dst->list); 148 list_add_tail(&data->list, &dst->list); 149 raw_spin_unlock_irqrestore(&dst->lock, flags); 150 151 /* 152 * The list addition should be visible before sending the IPI 153 * handler locks the list to pull the entry off it because of 154 * normal cache coherency rules implied by spinlocks. 155 * 156 * If IPIs can go out of order to the cache coherency protocol 157 * in an architecture, sufficient synchronisation should be added 158 * to arch code to make it appear to obey cache coherency WRT 159 * locking and barrier primitives. Generic code isn't really 160 * equipped to do the right thing... 161 */ 162 if (ipi) 163 arch_send_call_function_single_ipi(cpu); 164 165 if (wait) 166 csd_lock_wait(data); 167 } 168 169 /* 170 * Invoked by arch to handle an IPI for call function. Must be called with 171 * interrupts disabled. 172 */ 173 void generic_smp_call_function_interrupt(void) 174 { 175 struct call_function_data *data; 176 int cpu = smp_processor_id(); 177 178 /* 179 * Shouldn't receive this interrupt on a cpu that is not yet online. 180 */ 181 WARN_ON_ONCE(!cpu_online(cpu)); 182 183 /* 184 * Ensure entry is visible on call_function_queue after we have 185 * entered the IPI. See comment in smp_call_function_many. 186 * If we don't have this, then we may miss an entry on the list 187 * and never get another IPI to process it. 188 */ 189 smp_mb(); 190 191 /* 192 * It's ok to use list_for_each_rcu() here even though we may 193 * delete 'pos', since list_del_rcu() doesn't clear ->next 194 */ 195 list_for_each_entry_rcu(data, &call_function.queue, csd.list) { 196 int refs; 197 smp_call_func_t func; 198 199 /* 200 * Since we walk the list without any locks, we might 201 * see an entry that was completed, removed from the 202 * list and is in the process of being reused. 203 * 204 * We must check that the cpu is in the cpumask before 205 * checking the refs, and both must be set before 206 * executing the callback on this cpu. 207 */ 208 209 if (!cpumask_test_cpu(cpu, data->cpumask)) 210 continue; 211 212 smp_rmb(); 213 214 if (atomic_read(&data->refs) == 0) 215 continue; 216 217 func = data->csd.func; /* save for later warn */ 218 func(data->csd.info); 219 220 /* 221 * If the cpu mask is not still set then func enabled 222 * interrupts (BUG), and this cpu took another smp call 223 * function interrupt and executed func(info) twice 224 * on this cpu. That nested execution decremented refs. 225 */ 226 if (!cpumask_test_and_clear_cpu(cpu, data->cpumask)) { 227 WARN(1, "%pf enabled interrupts and double executed\n", func); 228 continue; 229 } 230 231 refs = atomic_dec_return(&data->refs); 232 WARN_ON(refs < 0); 233 234 if (refs) 235 continue; 236 237 WARN_ON(!cpumask_empty(data->cpumask)); 238 239 raw_spin_lock(&call_function.lock); 240 list_del_rcu(&data->csd.list); 241 raw_spin_unlock(&call_function.lock); 242 243 csd_unlock(&data->csd); 244 } 245 246 } 247 248 /* 249 * Invoked by arch to handle an IPI for call function single. Must be 250 * called from the arch with interrupts disabled. 251 */ 252 void generic_smp_call_function_single_interrupt(void) 253 { 254 struct call_single_queue *q = &__get_cpu_var(call_single_queue); 255 unsigned int data_flags; 256 LIST_HEAD(list); 257 258 /* 259 * Shouldn't receive this interrupt on a cpu that is not yet online. 260 */ 261 WARN_ON_ONCE(!cpu_online(smp_processor_id())); 262 263 raw_spin_lock(&q->lock); 264 list_replace_init(&q->list, &list); 265 raw_spin_unlock(&q->lock); 266 267 while (!list_empty(&list)) { 268 struct call_single_data *data; 269 270 data = list_entry(list.next, struct call_single_data, list); 271 list_del(&data->list); 272 273 /* 274 * 'data' can be invalid after this call if flags == 0 275 * (when called through generic_exec_single()), 276 * so save them away before making the call: 277 */ 278 data_flags = data->flags; 279 280 data->func(data->info); 281 282 /* 283 * Unlocked CSDs are valid through generic_exec_single(): 284 */ 285 if (data_flags & CSD_FLAG_LOCK) 286 csd_unlock(data); 287 } 288 } 289 290 static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_data, csd_data); 291 292 /* 293 * smp_call_function_single - Run a function on a specific CPU 294 * @func: The function to run. This must be fast and non-blocking. 295 * @info: An arbitrary pointer to pass to the function. 296 * @wait: If true, wait until function has completed on other CPUs. 297 * 298 * Returns 0 on success, else a negative status code. 299 */ 300 int smp_call_function_single(int cpu, smp_call_func_t func, void *info, 301 int wait) 302 { 303 struct call_single_data d = { 304 .flags = 0, 305 }; 306 unsigned long flags; 307 int this_cpu; 308 int err = 0; 309 310 /* 311 * prevent preemption and reschedule on another processor, 312 * as well as CPU removal 313 */ 314 this_cpu = get_cpu(); 315 316 /* 317 * Can deadlock when called with interrupts disabled. 318 * We allow cpu's that are not yet online though, as no one else can 319 * send smp call function interrupt to this cpu and as such deadlocks 320 * can't happen. 321 */ 322 WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled() 323 && !oops_in_progress); 324 325 if (cpu == this_cpu) { 326 local_irq_save(flags); 327 func(info); 328 local_irq_restore(flags); 329 } else { 330 if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) { 331 struct call_single_data *data = &d; 332 333 if (!wait) 334 data = &__get_cpu_var(csd_data); 335 336 csd_lock(data); 337 338 data->func = func; 339 data->info = info; 340 generic_exec_single(cpu, data, wait); 341 } else { 342 err = -ENXIO; /* CPU not online */ 343 } 344 } 345 346 put_cpu(); 347 348 return err; 349 } 350 EXPORT_SYMBOL(smp_call_function_single); 351 352 /* 353 * smp_call_function_any - Run a function on any of the given cpus 354 * @mask: The mask of cpus it can run on. 355 * @func: The function to run. This must be fast and non-blocking. 356 * @info: An arbitrary pointer to pass to the function. 357 * @wait: If true, wait until function has completed. 358 * 359 * Returns 0 on success, else a negative status code (if no cpus were online). 360 * Note that @wait will be implicitly turned on in case of allocation failures, 361 * since we fall back to on-stack allocation. 362 * 363 * Selection preference: 364 * 1) current cpu if in @mask 365 * 2) any cpu of current node if in @mask 366 * 3) any other online cpu in @mask 367 */ 368 int smp_call_function_any(const struct cpumask *mask, 369 smp_call_func_t func, void *info, int wait) 370 { 371 unsigned int cpu; 372 const struct cpumask *nodemask; 373 int ret; 374 375 /* Try for same CPU (cheapest) */ 376 cpu = get_cpu(); 377 if (cpumask_test_cpu(cpu, mask)) 378 goto call; 379 380 /* Try for same node. */ 381 nodemask = cpumask_of_node(cpu_to_node(cpu)); 382 for (cpu = cpumask_first_and(nodemask, mask); cpu < nr_cpu_ids; 383 cpu = cpumask_next_and(cpu, nodemask, mask)) { 384 if (cpu_online(cpu)) 385 goto call; 386 } 387 388 /* Any online will do: smp_call_function_single handles nr_cpu_ids. */ 389 cpu = cpumask_any_and(mask, cpu_online_mask); 390 call: 391 ret = smp_call_function_single(cpu, func, info, wait); 392 put_cpu(); 393 return ret; 394 } 395 EXPORT_SYMBOL_GPL(smp_call_function_any); 396 397 /** 398 * __smp_call_function_single(): Run a function on a specific CPU 399 * @cpu: The CPU to run on. 400 * @data: Pre-allocated and setup data structure 401 * @wait: If true, wait until function has completed on specified CPU. 402 * 403 * Like smp_call_function_single(), but allow caller to pass in a 404 * pre-allocated data structure. Useful for embedding @data inside 405 * other structures, for instance. 406 */ 407 void __smp_call_function_single(int cpu, struct call_single_data *data, 408 int wait) 409 { 410 unsigned int this_cpu; 411 unsigned long flags; 412 413 this_cpu = get_cpu(); 414 /* 415 * Can deadlock when called with interrupts disabled. 416 * We allow cpu's that are not yet online though, as no one else can 417 * send smp call function interrupt to this cpu and as such deadlocks 418 * can't happen. 419 */ 420 WARN_ON_ONCE(cpu_online(smp_processor_id()) && wait && irqs_disabled() 421 && !oops_in_progress); 422 423 if (cpu == this_cpu) { 424 local_irq_save(flags); 425 data->func(data->info); 426 local_irq_restore(flags); 427 } else { 428 csd_lock(data); 429 generic_exec_single(cpu, data, wait); 430 } 431 put_cpu(); 432 } 433 434 /** 435 * smp_call_function_many(): Run a function on a set of other CPUs. 436 * @mask: The set of cpus to run on (only runs on online subset). 437 * @func: The function to run. This must be fast and non-blocking. 438 * @info: An arbitrary pointer to pass to the function. 439 * @wait: If true, wait (atomically) until function has completed 440 * on other CPUs. 441 * 442 * If @wait is true, then returns once @func has returned. 443 * 444 * You must not call this function with disabled interrupts or from a 445 * hardware interrupt handler or from a bottom half handler. Preemption 446 * must be disabled when calling this function. 447 */ 448 void smp_call_function_many(const struct cpumask *mask, 449 smp_call_func_t func, void *info, bool wait) 450 { 451 struct call_function_data *data; 452 unsigned long flags; 453 int refs, cpu, next_cpu, this_cpu = smp_processor_id(); 454 455 /* 456 * Can deadlock when called with interrupts disabled. 457 * We allow cpu's that are not yet online though, as no one else can 458 * send smp call function interrupt to this cpu and as such deadlocks 459 * can't happen. 460 */ 461 WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled() 462 && !oops_in_progress && !early_boot_irqs_disabled); 463 464 /* Try to fastpath. So, what's a CPU they want? Ignoring this one. */ 465 cpu = cpumask_first_and(mask, cpu_online_mask); 466 if (cpu == this_cpu) 467 cpu = cpumask_next_and(cpu, mask, cpu_online_mask); 468 469 /* No online cpus? We're done. */ 470 if (cpu >= nr_cpu_ids) 471 return; 472 473 /* Do we have another CPU which isn't us? */ 474 next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask); 475 if (next_cpu == this_cpu) 476 next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask); 477 478 /* Fastpath: do that cpu by itself. */ 479 if (next_cpu >= nr_cpu_ids) { 480 smp_call_function_single(cpu, func, info, wait); 481 return; 482 } 483 484 data = &__get_cpu_var(cfd_data); 485 csd_lock(&data->csd); 486 487 /* This BUG_ON verifies our reuse assertions and can be removed */ 488 BUG_ON(atomic_read(&data->refs) || !cpumask_empty(data->cpumask)); 489 490 /* 491 * The global call function queue list add and delete are protected 492 * by a lock, but the list is traversed without any lock, relying 493 * on the rcu list add and delete to allow safe concurrent traversal. 494 * We reuse the call function data without waiting for any grace 495 * period after some other cpu removes it from the global queue. 496 * This means a cpu might find our data block as it is being 497 * filled out. 498 * 499 * We hold off the interrupt handler on the other cpu by 500 * ordering our writes to the cpu mask vs our setting of the 501 * refs counter. We assert only the cpu owning the data block 502 * will set a bit in cpumask, and each bit will only be cleared 503 * by the subject cpu. Each cpu must first find its bit is 504 * set and then check that refs is set indicating the element is 505 * ready to be processed, otherwise it must skip the entry. 506 * 507 * On the previous iteration refs was set to 0 by another cpu. 508 * To avoid the use of transitivity, set the counter to 0 here 509 * so the wmb will pair with the rmb in the interrupt handler. 510 */ 511 atomic_set(&data->refs, 0); /* convert 3rd to 1st party write */ 512 513 data->csd.func = func; 514 data->csd.info = info; 515 516 /* Ensure 0 refs is visible before mask. Also orders func and info */ 517 smp_wmb(); 518 519 /* We rely on the "and" being processed before the store */ 520 cpumask_and(data->cpumask, mask, cpu_online_mask); 521 cpumask_clear_cpu(this_cpu, data->cpumask); 522 refs = cpumask_weight(data->cpumask); 523 524 /* Some callers race with other cpus changing the passed mask */ 525 if (unlikely(!refs)) { 526 csd_unlock(&data->csd); 527 return; 528 } 529 530 raw_spin_lock_irqsave(&call_function.lock, flags); 531 /* 532 * Place entry at the _HEAD_ of the list, so that any cpu still 533 * observing the entry in generic_smp_call_function_interrupt() 534 * will not miss any other list entries: 535 */ 536 list_add_rcu(&data->csd.list, &call_function.queue); 537 /* 538 * We rely on the wmb() in list_add_rcu to complete our writes 539 * to the cpumask before this write to refs, which indicates 540 * data is on the list and is ready to be processed. 541 */ 542 atomic_set(&data->refs, refs); 543 raw_spin_unlock_irqrestore(&call_function.lock, flags); 544 545 /* 546 * Make the list addition visible before sending the ipi. 547 * (IPIs must obey or appear to obey normal Linux cache 548 * coherency rules -- see comment in generic_exec_single). 549 */ 550 smp_mb(); 551 552 /* Send a message to all CPUs in the map */ 553 arch_send_call_function_ipi_mask(data->cpumask); 554 555 /* Optionally wait for the CPUs to complete */ 556 if (wait) 557 csd_lock_wait(&data->csd); 558 } 559 EXPORT_SYMBOL(smp_call_function_many); 560 561 /** 562 * smp_call_function(): Run a function on all other CPUs. 563 * @func: The function to run. This must be fast and non-blocking. 564 * @info: An arbitrary pointer to pass to the function. 565 * @wait: If true, wait (atomically) until function has completed 566 * on other CPUs. 567 * 568 * Returns 0. 569 * 570 * If @wait is true, then returns once @func has returned; otherwise 571 * it returns just before the target cpu calls @func. 572 * 573 * You must not call this function with disabled interrupts or from a 574 * hardware interrupt handler or from a bottom half handler. 575 */ 576 int smp_call_function(smp_call_func_t func, void *info, int wait) 577 { 578 preempt_disable(); 579 smp_call_function_many(cpu_online_mask, func, info, wait); 580 preempt_enable(); 581 582 return 0; 583 } 584 EXPORT_SYMBOL(smp_call_function); 585 586 void ipi_call_lock(void) 587 { 588 raw_spin_lock(&call_function.lock); 589 } 590 591 void ipi_call_unlock(void) 592 { 593 raw_spin_unlock(&call_function.lock); 594 } 595 596 void ipi_call_lock_irq(void) 597 { 598 raw_spin_lock_irq(&call_function.lock); 599 } 600 601 void ipi_call_unlock_irq(void) 602 { 603 raw_spin_unlock_irq(&call_function.lock); 604 } 605 #endif /* USE_GENERIC_SMP_HELPERS */ 606 607 /* Setup configured maximum number of CPUs to activate */ 608 unsigned int setup_max_cpus = NR_CPUS; 609 EXPORT_SYMBOL(setup_max_cpus); 610 611 612 /* 613 * Setup routine for controlling SMP activation 614 * 615 * Command-line option of "nosmp" or "maxcpus=0" will disable SMP 616 * activation entirely (the MPS table probe still happens, though). 617 * 618 * Command-line option of "maxcpus=<NUM>", where <NUM> is an integer 619 * greater than 0, limits the maximum number of CPUs activated in 620 * SMP mode to <NUM>. 621 */ 622 623 void __weak arch_disable_smp_support(void) { } 624 625 static int __init nosmp(char *str) 626 { 627 setup_max_cpus = 0; 628 arch_disable_smp_support(); 629 630 return 0; 631 } 632 633 early_param("nosmp", nosmp); 634 635 /* this is hard limit */ 636 static int __init nrcpus(char *str) 637 { 638 int nr_cpus; 639 640 get_option(&str, &nr_cpus); 641 if (nr_cpus > 0 && nr_cpus < nr_cpu_ids) 642 nr_cpu_ids = nr_cpus; 643 644 return 0; 645 } 646 647 early_param("nr_cpus", nrcpus); 648 649 static int __init maxcpus(char *str) 650 { 651 get_option(&str, &setup_max_cpus); 652 if (setup_max_cpus == 0) 653 arch_disable_smp_support(); 654 655 return 0; 656 } 657 658 early_param("maxcpus", maxcpus); 659 660 /* Setup number of possible processor ids */ 661 int nr_cpu_ids __read_mostly = NR_CPUS; 662 EXPORT_SYMBOL(nr_cpu_ids); 663 664 /* An arch may set nr_cpu_ids earlier if needed, so this would be redundant */ 665 void __init setup_nr_cpu_ids(void) 666 { 667 nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1; 668 } 669 670 /* Called by boot processor to activate the rest. */ 671 void __init smp_init(void) 672 { 673 unsigned int cpu; 674 675 /* FIXME: This should be done in userspace --RR */ 676 for_each_present_cpu(cpu) { 677 if (num_online_cpus() >= setup_max_cpus) 678 break; 679 if (!cpu_online(cpu)) 680 cpu_up(cpu); 681 } 682 683 /* Any cleanup work */ 684 printk(KERN_INFO "Brought up %ld CPUs\n", (long)num_online_cpus()); 685 smp_cpus_done(setup_max_cpus); 686 } 687 688 /* 689 * Call a function on all processors. May be used during early boot while 690 * early_boot_irqs_disabled is set. Use local_irq_save/restore() instead 691 * of local_irq_disable/enable(). 692 */ 693 int on_each_cpu(void (*func) (void *info), void *info, int wait) 694 { 695 unsigned long flags; 696 int ret = 0; 697 698 preempt_disable(); 699 ret = smp_call_function(func, info, wait); 700 local_irq_save(flags); 701 func(info); 702 local_irq_restore(flags); 703 preempt_enable(); 704 return ret; 705 } 706 EXPORT_SYMBOL(on_each_cpu); 707