1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 * 5 * Derived from MIPS: 6 * Copyright (C) 2000, 2001 Kanoj Sarcar 7 * Copyright (C) 2000, 2001 Ralf Baechle 8 * Copyright (C) 2000, 2001 Silicon Graphics, Inc. 9 * Copyright (C) 2000, 2001, 2003 Broadcom Corporation 10 */ 11 #include <linux/cpu.h> 12 #include <linux/cpumask.h> 13 #include <linux/init.h> 14 #include <linux/interrupt.h> 15 #include <linux/seq_file.h> 16 #include <linux/smp.h> 17 #include <linux/threads.h> 18 #include <linux/export.h> 19 #include <linux/time.h> 20 #include <linux/tracepoint.h> 21 #include <linux/sched/hotplug.h> 22 #include <linux/sched/task_stack.h> 23 24 #include <asm/cpu.h> 25 #include <asm/idle.h> 26 #include <asm/loongson.h> 27 #include <asm/mmu_context.h> 28 #include <asm/numa.h> 29 #include <asm/processor.h> 30 #include <asm/setup.h> 31 #include <asm/time.h> 32 33 int __cpu_number_map[NR_CPUS]; /* Map physical to logical */ 34 EXPORT_SYMBOL(__cpu_number_map); 35 36 int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */ 37 EXPORT_SYMBOL(__cpu_logical_map); 38 39 /* Number of threads (siblings) per CPU core */ 40 int smp_num_siblings = 1; 41 EXPORT_SYMBOL(smp_num_siblings); 42 43 /* Representing the threads (siblings) of each logical CPU */ 44 cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly; 45 EXPORT_SYMBOL(cpu_sibling_map); 46 47 /* Representing the core map of multi-core chips of each logical CPU */ 48 cpumask_t cpu_core_map[NR_CPUS] __read_mostly; 49 EXPORT_SYMBOL(cpu_core_map); 50 51 static DECLARE_COMPLETION(cpu_starting); 52 static DECLARE_COMPLETION(cpu_running); 53 54 /* 55 * A logcal cpu mask containing only one VPE per core to 56 * reduce the number of IPIs on large MT systems. 57 */ 58 cpumask_t cpu_foreign_map[NR_CPUS] __read_mostly; 59 EXPORT_SYMBOL(cpu_foreign_map); 60 61 /* representing cpus for which sibling maps can be computed */ 62 static cpumask_t cpu_sibling_setup_map; 63 64 /* representing cpus for which core maps can be computed */ 65 static cpumask_t cpu_core_setup_map; 66 67 struct secondary_data cpuboot_data; 68 static DEFINE_PER_CPU(int, cpu_state); 69 DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat); 70 EXPORT_PER_CPU_SYMBOL(irq_stat); 71 72 enum ipi_msg_type { 73 IPI_RESCHEDULE, 74 IPI_CALL_FUNCTION, 75 }; 76 77 static const char *ipi_types[NR_IPI] __tracepoint_string = { 78 [IPI_RESCHEDULE] = "Rescheduling interrupts", 79 [IPI_CALL_FUNCTION] = "Function call interrupts", 80 }; 81 82 void show_ipi_list(struct seq_file *p, int prec) 83 { 84 unsigned int cpu, i; 85 86 for (i = 0; i < NR_IPI; i++) { 87 seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, prec >= 4 ? " " : ""); 88 for_each_online_cpu(cpu) 89 seq_printf(p, "%10u ", per_cpu(irq_stat, cpu).ipi_irqs[i]); 90 seq_printf(p, " LoongArch %d %s\n", i + 1, ipi_types[i]); 91 } 92 } 93 94 /* Send mailbox buffer via Mail_Send */ 95 static void csr_mail_send(uint64_t data, int cpu, int mailbox) 96 { 97 uint64_t val; 98 99 /* Send high 32 bits */ 100 val = IOCSR_MBUF_SEND_BLOCKING; 101 val |= (IOCSR_MBUF_SEND_BOX_HI(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT); 102 val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT); 103 val |= (data & IOCSR_MBUF_SEND_H32_MASK); 104 iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND); 105 106 /* Send low 32 bits */ 107 val = IOCSR_MBUF_SEND_BLOCKING; 108 val |= (IOCSR_MBUF_SEND_BOX_LO(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT); 109 val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT); 110 val |= (data << IOCSR_MBUF_SEND_BUF_SHIFT); 111 iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND); 112 }; 113 114 static u32 ipi_read_clear(int cpu) 115 { 116 u32 action; 117 118 /* Load the ipi register to figure out what we're supposed to do */ 119 action = iocsr_read32(LOONGARCH_IOCSR_IPI_STATUS); 120 /* Clear the ipi register to clear the interrupt */ 121 iocsr_write32(action, LOONGARCH_IOCSR_IPI_CLEAR); 122 smp_mb(); 123 124 return action; 125 } 126 127 static void ipi_write_action(int cpu, u32 action) 128 { 129 unsigned int irq = 0; 130 131 while ((irq = ffs(action))) { 132 uint32_t val = IOCSR_IPI_SEND_BLOCKING; 133 134 val |= (irq - 1); 135 val |= (cpu << IOCSR_IPI_SEND_CPU_SHIFT); 136 iocsr_write32(val, LOONGARCH_IOCSR_IPI_SEND); 137 action &= ~BIT(irq - 1); 138 } 139 } 140 141 void loongson3_send_ipi_single(int cpu, unsigned int action) 142 { 143 ipi_write_action(cpu_logical_map(cpu), (u32)action); 144 } 145 146 void loongson3_send_ipi_mask(const struct cpumask *mask, unsigned int action) 147 { 148 unsigned int i; 149 150 for_each_cpu(i, mask) 151 ipi_write_action(cpu_logical_map(i), (u32)action); 152 } 153 154 irqreturn_t loongson3_ipi_interrupt(int irq, void *dev) 155 { 156 unsigned int action; 157 unsigned int cpu = smp_processor_id(); 158 159 action = ipi_read_clear(cpu_logical_map(cpu)); 160 161 if (action & SMP_RESCHEDULE) { 162 scheduler_ipi(); 163 per_cpu(irq_stat, cpu).ipi_irqs[IPI_RESCHEDULE]++; 164 } 165 166 if (action & SMP_CALL_FUNCTION) { 167 generic_smp_call_function_interrupt(); 168 per_cpu(irq_stat, cpu).ipi_irqs[IPI_CALL_FUNCTION]++; 169 } 170 171 return IRQ_HANDLED; 172 } 173 174 void __init loongson3_smp_setup(void) 175 { 176 cpu_data[0].core = cpu_logical_map(0) % loongson_sysconf.cores_per_package; 177 cpu_data[0].package = cpu_logical_map(0) / loongson_sysconf.cores_per_package; 178 179 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 180 pr_info("Detected %i available CPU(s)\n", loongson_sysconf.nr_cpus); 181 } 182 183 void __init loongson3_prepare_cpus(unsigned int max_cpus) 184 { 185 int i = 0; 186 187 for (i = 0; i < loongson_sysconf.nr_cpus; i++) { 188 set_cpu_present(i, true); 189 csr_mail_send(0, __cpu_logical_map[i], 0); 190 } 191 192 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; 193 } 194 195 /* 196 * Setup the PC, SP, and TP of a secondary processor and start it running! 197 */ 198 void loongson3_boot_secondary(int cpu, struct task_struct *idle) 199 { 200 unsigned long entry; 201 202 pr_info("Booting CPU#%d...\n", cpu); 203 204 entry = __pa_symbol((unsigned long)&smpboot_entry); 205 cpuboot_data.stack = (unsigned long)__KSTK_TOS(idle); 206 cpuboot_data.thread_info = (unsigned long)task_thread_info(idle); 207 208 csr_mail_send(entry, cpu_logical_map(cpu), 0); 209 210 loongson3_send_ipi_single(cpu, SMP_BOOT_CPU); 211 } 212 213 /* 214 * SMP init and finish on secondary CPUs 215 */ 216 void loongson3_init_secondary(void) 217 { 218 unsigned int cpu = smp_processor_id(); 219 unsigned int imask = ECFGF_IP0 | ECFGF_IP1 | ECFGF_IP2 | 220 ECFGF_IPI | ECFGF_PMC | ECFGF_TIMER; 221 222 change_csr_ecfg(ECFG0_IM, imask); 223 224 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 225 226 #ifdef CONFIG_NUMA 227 numa_add_cpu(cpu); 228 #endif 229 per_cpu(cpu_state, cpu) = CPU_ONLINE; 230 cpu_data[cpu].core = 231 cpu_logical_map(cpu) % loongson_sysconf.cores_per_package; 232 cpu_data[cpu].package = 233 cpu_logical_map(cpu) / loongson_sysconf.cores_per_package; 234 } 235 236 void loongson3_smp_finish(void) 237 { 238 local_irq_enable(); 239 iocsr_write64(0, LOONGARCH_IOCSR_MBUF0); 240 pr_info("CPU#%d finished\n", smp_processor_id()); 241 } 242 243 #ifdef CONFIG_HOTPLUG_CPU 244 245 static bool io_master(int cpu) 246 { 247 if (cpu == 0) 248 return true; 249 250 return false; 251 } 252 253 int loongson3_cpu_disable(void) 254 { 255 unsigned long flags; 256 unsigned int cpu = smp_processor_id(); 257 258 if (io_master(cpu)) 259 return -EBUSY; 260 261 #ifdef CONFIG_NUMA 262 numa_remove_cpu(cpu); 263 #endif 264 set_cpu_online(cpu, false); 265 calculate_cpu_foreign_map(); 266 local_irq_save(flags); 267 irq_migrate_all_off_this_cpu(); 268 clear_csr_ecfg(ECFG0_IM); 269 local_irq_restore(flags); 270 local_flush_tlb_all(); 271 272 return 0; 273 } 274 275 void loongson3_cpu_die(unsigned int cpu) 276 { 277 while (per_cpu(cpu_state, cpu) != CPU_DEAD) 278 cpu_relax(); 279 280 mb(); 281 } 282 283 /* 284 * The target CPU should go to XKPRANGE (uncached area) and flush 285 * ICache/DCache/VCache before the control CPU can safely disable its clock. 286 */ 287 static void loongson3_play_dead(int *state_addr) 288 { 289 register int val; 290 register void *addr; 291 register void (*init_fn)(void); 292 293 __asm__ __volatile__( 294 " li.d %[addr], 0x8000000000000000\n" 295 "1: cacop 0x8, %[addr], 0 \n" /* flush ICache */ 296 " cacop 0x8, %[addr], 1 \n" 297 " cacop 0x8, %[addr], 2 \n" 298 " cacop 0x8, %[addr], 3 \n" 299 " cacop 0x9, %[addr], 0 \n" /* flush DCache */ 300 " cacop 0x9, %[addr], 1 \n" 301 " cacop 0x9, %[addr], 2 \n" 302 " cacop 0x9, %[addr], 3 \n" 303 " addi.w %[sets], %[sets], -1 \n" 304 " addi.d %[addr], %[addr], 0x40 \n" 305 " bnez %[sets], 1b \n" 306 " li.d %[addr], 0x8000000000000000\n" 307 "2: cacop 0xa, %[addr], 0 \n" /* flush VCache */ 308 " cacop 0xa, %[addr], 1 \n" 309 " cacop 0xa, %[addr], 2 \n" 310 " cacop 0xa, %[addr], 3 \n" 311 " cacop 0xa, %[addr], 4 \n" 312 " cacop 0xa, %[addr], 5 \n" 313 " cacop 0xa, %[addr], 6 \n" 314 " cacop 0xa, %[addr], 7 \n" 315 " cacop 0xa, %[addr], 8 \n" 316 " cacop 0xa, %[addr], 9 \n" 317 " cacop 0xa, %[addr], 10 \n" 318 " cacop 0xa, %[addr], 11 \n" 319 " cacop 0xa, %[addr], 12 \n" 320 " cacop 0xa, %[addr], 13 \n" 321 " cacop 0xa, %[addr], 14 \n" 322 " cacop 0xa, %[addr], 15 \n" 323 " addi.w %[vsets], %[vsets], -1 \n" 324 " addi.d %[addr], %[addr], 0x40 \n" 325 " bnez %[vsets], 2b \n" 326 " li.w %[val], 0x7 \n" /* *state_addr = CPU_DEAD; */ 327 " st.w %[val], %[state_addr], 0 \n" 328 " dbar 0 \n" 329 " cacop 0x11, %[state_addr], 0 \n" /* flush entry of *state_addr */ 330 : [addr] "=&r" (addr), [val] "=&r" (val) 331 : [state_addr] "r" (state_addr), 332 [sets] "r" (cpu_data[smp_processor_id()].dcache.sets), 333 [vsets] "r" (cpu_data[smp_processor_id()].vcache.sets)); 334 335 local_irq_enable(); 336 change_csr_ecfg(ECFG0_IM, ECFGF_IPI); 337 338 __asm__ __volatile__( 339 " idle 0 \n" 340 " li.w $t0, 0x1020 \n" 341 " iocsrrd.d %[init_fn], $t0 \n" /* Get init PC */ 342 : [init_fn] "=&r" (addr) 343 : /* No Input */ 344 : "a0"); 345 init_fn = __va(addr); 346 347 init_fn(); 348 unreachable(); 349 } 350 351 void play_dead(void) 352 { 353 int *state_addr; 354 unsigned int cpu = smp_processor_id(); 355 void (*play_dead_uncached)(int *s); 356 357 idle_task_exit(); 358 play_dead_uncached = (void *)TO_UNCACHE(__pa((unsigned long)loongson3_play_dead)); 359 state_addr = &per_cpu(cpu_state, cpu); 360 mb(); 361 play_dead_uncached(state_addr); 362 } 363 364 static int loongson3_enable_clock(unsigned int cpu) 365 { 366 uint64_t core_id = cpu_data[cpu].core; 367 uint64_t package_id = cpu_data[cpu].package; 368 369 LOONGSON_FREQCTRL(package_id) |= 1 << (core_id * 4 + 3); 370 371 return 0; 372 } 373 374 static int loongson3_disable_clock(unsigned int cpu) 375 { 376 uint64_t core_id = cpu_data[cpu].core; 377 uint64_t package_id = cpu_data[cpu].package; 378 379 LOONGSON_FREQCTRL(package_id) &= ~(1 << (core_id * 4 + 3)); 380 381 return 0; 382 } 383 384 static int register_loongson3_notifier(void) 385 { 386 return cpuhp_setup_state_nocalls(CPUHP_LOONGARCH_SOC_PREPARE, 387 "loongarch/loongson:prepare", 388 loongson3_enable_clock, 389 loongson3_disable_clock); 390 } 391 early_initcall(register_loongson3_notifier); 392 393 #endif 394 395 /* 396 * Power management 397 */ 398 #ifdef CONFIG_PM 399 400 static int loongson3_ipi_suspend(void) 401 { 402 return 0; 403 } 404 405 static void loongson3_ipi_resume(void) 406 { 407 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 408 } 409 410 static struct syscore_ops loongson3_ipi_syscore_ops = { 411 .resume = loongson3_ipi_resume, 412 .suspend = loongson3_ipi_suspend, 413 }; 414 415 /* 416 * Enable boot cpu ipi before enabling nonboot cpus 417 * during syscore_resume. 418 */ 419 static int __init ipi_pm_init(void) 420 { 421 register_syscore_ops(&loongson3_ipi_syscore_ops); 422 return 0; 423 } 424 425 core_initcall(ipi_pm_init); 426 #endif 427 428 static inline void set_cpu_sibling_map(int cpu) 429 { 430 int i; 431 432 cpumask_set_cpu(cpu, &cpu_sibling_setup_map); 433 434 if (smp_num_siblings <= 1) 435 cpumask_set_cpu(cpu, &cpu_sibling_map[cpu]); 436 else { 437 for_each_cpu(i, &cpu_sibling_setup_map) { 438 if (cpus_are_siblings(cpu, i)) { 439 cpumask_set_cpu(i, &cpu_sibling_map[cpu]); 440 cpumask_set_cpu(cpu, &cpu_sibling_map[i]); 441 } 442 } 443 } 444 } 445 446 static inline void set_cpu_core_map(int cpu) 447 { 448 int i; 449 450 cpumask_set_cpu(cpu, &cpu_core_setup_map); 451 452 for_each_cpu(i, &cpu_core_setup_map) { 453 if (cpu_data[cpu].package == cpu_data[i].package) { 454 cpumask_set_cpu(i, &cpu_core_map[cpu]); 455 cpumask_set_cpu(cpu, &cpu_core_map[i]); 456 } 457 } 458 } 459 460 /* 461 * Calculate a new cpu_foreign_map mask whenever a 462 * new cpu appears or disappears. 463 */ 464 void calculate_cpu_foreign_map(void) 465 { 466 int i, k, core_present; 467 cpumask_t temp_foreign_map; 468 469 /* Re-calculate the mask */ 470 cpumask_clear(&temp_foreign_map); 471 for_each_online_cpu(i) { 472 core_present = 0; 473 for_each_cpu(k, &temp_foreign_map) 474 if (cpus_are_siblings(i, k)) 475 core_present = 1; 476 if (!core_present) 477 cpumask_set_cpu(i, &temp_foreign_map); 478 } 479 480 for_each_online_cpu(i) 481 cpumask_andnot(&cpu_foreign_map[i], 482 &temp_foreign_map, &cpu_sibling_map[i]); 483 } 484 485 /* Preload SMP state for boot cpu */ 486 void smp_prepare_boot_cpu(void) 487 { 488 unsigned int cpu, node, rr_node; 489 490 set_cpu_possible(0, true); 491 set_cpu_online(0, true); 492 set_my_cpu_offset(per_cpu_offset(0)); 493 494 rr_node = first_node(node_online_map); 495 for_each_possible_cpu(cpu) { 496 node = early_cpu_to_node(cpu); 497 498 /* 499 * The mapping between present cpus and nodes has been 500 * built during MADT and SRAT parsing. 501 * 502 * If possible cpus = present cpus here, early_cpu_to_node 503 * will return valid node. 504 * 505 * If possible cpus > present cpus here (e.g. some possible 506 * cpus will be added by cpu-hotplug later), for possible but 507 * not present cpus, early_cpu_to_node will return NUMA_NO_NODE, 508 * and we just map them to online nodes in round-robin way. 509 * Once hotplugged, new correct mapping will be built for them. 510 */ 511 if (node != NUMA_NO_NODE) 512 set_cpu_numa_node(cpu, node); 513 else { 514 set_cpu_numa_node(cpu, rr_node); 515 rr_node = next_node_in(rr_node, node_online_map); 516 } 517 } 518 } 519 520 /* called from main before smp_init() */ 521 void __init smp_prepare_cpus(unsigned int max_cpus) 522 { 523 init_new_context(current, &init_mm); 524 current_thread_info()->cpu = 0; 525 loongson3_prepare_cpus(max_cpus); 526 set_cpu_sibling_map(0); 527 set_cpu_core_map(0); 528 calculate_cpu_foreign_map(); 529 #ifndef CONFIG_HOTPLUG_CPU 530 init_cpu_present(cpu_possible_mask); 531 #endif 532 } 533 534 int __cpu_up(unsigned int cpu, struct task_struct *tidle) 535 { 536 loongson3_boot_secondary(cpu, tidle); 537 538 /* Wait for CPU to start and be ready to sync counters */ 539 if (!wait_for_completion_timeout(&cpu_starting, 540 msecs_to_jiffies(5000))) { 541 pr_crit("CPU%u: failed to start\n", cpu); 542 return -EIO; 543 } 544 545 /* Wait for CPU to finish startup & mark itself online before return */ 546 wait_for_completion(&cpu_running); 547 548 return 0; 549 } 550 551 /* 552 * First C code run on the secondary CPUs after being started up by 553 * the master. 554 */ 555 asmlinkage void start_secondary(void) 556 { 557 unsigned int cpu; 558 559 sync_counter(); 560 cpu = smp_processor_id(); 561 set_my_cpu_offset(per_cpu_offset(cpu)); 562 563 cpu_probe(); 564 constant_clockevent_init(); 565 loongson3_init_secondary(); 566 567 set_cpu_sibling_map(cpu); 568 set_cpu_core_map(cpu); 569 570 notify_cpu_starting(cpu); 571 572 /* Notify boot CPU that we're starting */ 573 complete(&cpu_starting); 574 575 /* The CPU is running, now mark it online */ 576 set_cpu_online(cpu, true); 577 578 calculate_cpu_foreign_map(); 579 580 /* 581 * Notify boot CPU that we're up & online and it can safely return 582 * from __cpu_up() 583 */ 584 complete(&cpu_running); 585 586 /* 587 * irq will be enabled in loongson3_smp_finish(), enabling it too 588 * early is dangerous. 589 */ 590 WARN_ON_ONCE(!irqs_disabled()); 591 loongson3_smp_finish(); 592 593 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); 594 } 595 596 void __init smp_cpus_done(unsigned int max_cpus) 597 { 598 } 599 600 static void stop_this_cpu(void *dummy) 601 { 602 set_cpu_online(smp_processor_id(), false); 603 calculate_cpu_foreign_map(); 604 local_irq_disable(); 605 while (true); 606 } 607 608 void smp_send_stop(void) 609 { 610 smp_call_function(stop_this_cpu, NULL, 0); 611 } 612 613 int setup_profiling_timer(unsigned int multiplier) 614 { 615 return 0; 616 } 617 618 static void flush_tlb_all_ipi(void *info) 619 { 620 local_flush_tlb_all(); 621 } 622 623 void flush_tlb_all(void) 624 { 625 on_each_cpu(flush_tlb_all_ipi, NULL, 1); 626 } 627 628 static void flush_tlb_mm_ipi(void *mm) 629 { 630 local_flush_tlb_mm((struct mm_struct *)mm); 631 } 632 633 void flush_tlb_mm(struct mm_struct *mm) 634 { 635 if (atomic_read(&mm->mm_users) == 0) 636 return; /* happens as a result of exit_mmap() */ 637 638 preempt_disable(); 639 640 if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { 641 on_each_cpu_mask(mm_cpumask(mm), flush_tlb_mm_ipi, mm, 1); 642 } else { 643 unsigned int cpu; 644 645 for_each_online_cpu(cpu) { 646 if (cpu != smp_processor_id() && cpu_context(cpu, mm)) 647 cpu_context(cpu, mm) = 0; 648 } 649 local_flush_tlb_mm(mm); 650 } 651 652 preempt_enable(); 653 } 654 655 struct flush_tlb_data { 656 struct vm_area_struct *vma; 657 unsigned long addr1; 658 unsigned long addr2; 659 }; 660 661 static void flush_tlb_range_ipi(void *info) 662 { 663 struct flush_tlb_data *fd = info; 664 665 local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2); 666 } 667 668 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) 669 { 670 struct mm_struct *mm = vma->vm_mm; 671 672 preempt_disable(); 673 if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { 674 struct flush_tlb_data fd = { 675 .vma = vma, 676 .addr1 = start, 677 .addr2 = end, 678 }; 679 680 on_each_cpu_mask(mm_cpumask(mm), flush_tlb_range_ipi, &fd, 1); 681 } else { 682 unsigned int cpu; 683 684 for_each_online_cpu(cpu) { 685 if (cpu != smp_processor_id() && cpu_context(cpu, mm)) 686 cpu_context(cpu, mm) = 0; 687 } 688 local_flush_tlb_range(vma, start, end); 689 } 690 preempt_enable(); 691 } 692 693 static void flush_tlb_kernel_range_ipi(void *info) 694 { 695 struct flush_tlb_data *fd = info; 696 697 local_flush_tlb_kernel_range(fd->addr1, fd->addr2); 698 } 699 700 void flush_tlb_kernel_range(unsigned long start, unsigned long end) 701 { 702 struct flush_tlb_data fd = { 703 .addr1 = start, 704 .addr2 = end, 705 }; 706 707 on_each_cpu(flush_tlb_kernel_range_ipi, &fd, 1); 708 } 709 710 static void flush_tlb_page_ipi(void *info) 711 { 712 struct flush_tlb_data *fd = info; 713 714 local_flush_tlb_page(fd->vma, fd->addr1); 715 } 716 717 void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 718 { 719 preempt_disable(); 720 if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) { 721 struct flush_tlb_data fd = { 722 .vma = vma, 723 .addr1 = page, 724 }; 725 726 on_each_cpu_mask(mm_cpumask(vma->vm_mm), flush_tlb_page_ipi, &fd, 1); 727 } else { 728 unsigned int cpu; 729 730 for_each_online_cpu(cpu) { 731 if (cpu != smp_processor_id() && cpu_context(cpu, vma->vm_mm)) 732 cpu_context(cpu, vma->vm_mm) = 0; 733 } 734 local_flush_tlb_page(vma, page); 735 } 736 preempt_enable(); 737 } 738 EXPORT_SYMBOL(flush_tlb_page); 739 740 static void flush_tlb_one_ipi(void *info) 741 { 742 unsigned long vaddr = (unsigned long) info; 743 744 local_flush_tlb_one(vaddr); 745 } 746 747 void flush_tlb_one(unsigned long vaddr) 748 { 749 on_each_cpu(flush_tlb_one_ipi, (void *)vaddr, 1); 750 } 751 EXPORT_SYMBOL(flush_tlb_one); 752